From 67793470ab2b2bada7b287a7030271723ffed453 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 29 Oct 2024 15:04:58 +0100 Subject: [PATCH 001/107] feat(street): add street project draft fixture --- .../tests/fixtures/projectDrafts/street.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 mapswipe_workers/tests/fixtures/projectDrafts/street.json diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json new file mode 100644 index 000000000..6bc291e33 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -0,0 +1,23 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { "type": "Polygon", "coordinates": [ [ [ 39.0085, -6.7811 ], [ 39.0214, -6.7908 ], [ 39.0374, -6.7918 ], [ 39.0537, -6.7968 ], [ 39.0628, -6.8525 ], [ 39.0909, -6.8757 ], [ 39.1018, -6.8747 ], [ 39.1161, -6.8781 ], [ 39.1088, -6.9151 ], [ 39.089, -6.9307 ], [ 39.0864, -6.94 ], [ 39.0596, -6.9731 ], [ 39.0716, -6.9835 ], [ 39.0499, -7.0279 ], [ 39.0236, -7.0322 ], [ 39.038, -7.0499 ], [ 39.0693, -7.0328 ], [ 39.0826, -7.0671 ], [ 39.112, -7.0728 ], [ 39.1154, -7.0921 ], [ 39.1543, -7.0183 ], [ 39.1746, -7.0101 ], [ 39.1699, -6.9971 ], [ 39.196, -6.9848 ], [ 39.2163, -6.9934 ], [ 39.2502, -6.9509 ], [ 39.2768, -6.9486 ], [ 39.2881, -6.9529 ], [ 39.3136, -6.9595 ], [ 39.3276, -6.9483 ], [ 39.3686, -6.9726 ], [ 39.3826, -6.9677 ], [ 39.393, -7.0 ], [ 39.4038, -7.0284 ], [ 39.4037, -7.0379 ], [ 39.4028, -7.0741 ], [ 39.4, -7.0914 ], [ 39.4154, -7.1286 ], [ 39.4049, -7.1595 ], [ 39.4124, -7.1661 ], [ 39.4287, -7.1546 ], [ 39.4395, -7.1618 ], [ 39.4501, -7.1827 ], [ 39.4897, -7.1692 ], [ 39.51, -7.1833 ], [ 39.5378, -7.1229 ], [ 39.5529, -7.1335 ], [ 39.5453, -7.0318 ], [ 39.5376, -6.9873 ], [ 39.4976, -6.9093 ], [ 39.4717, -6.8731 ], [ 39.3557, -6.8455 ], [ 39.3205, -6.8175 ], [ 39.3074, -6.8193 ], [ 39.3002, -6.8253 ], [ 39.293, -6.8317 ], [ 39.2944, -6.818 ], [ 39.2963, -6.8084 ], [ 39.2855, -6.7859 ], [ 39.2808, -6.7347 ], [ 39.2609, -6.7602 ], [ 39.2344, -6.715 ], [ 39.2004, -6.6376 ], [ 39.165, -6.6022 ], [ 39.1353, -6.5748 ], [ 39.1215, -6.5662 ], [ 39.0945, -6.5987 ], [ 39.086, -6.5956 ], [ 39.0803, -6.6062 ], [ 39.0578, -6.7139 ], [ 39.059, -6.723 ], [ 39.0085, -6.7811 ] ], [ [ 39.301, -6.8372 ], [ 39.3048, -6.8555 ], [ 39.2967, -6.8469 ], [ 39.301, -6.8372 ] ], [ [ 39.3048, -6.8555 ], [ 39.2949, -6.8707 ], [ 39.2921, -6.8607 ], [ 39.3048, -6.8555 ] ] ] } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25 +} From 40cdc667d06b6ba112eeb67bf406a238542e6d3d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 29 Oct 2024 17:07:15 +0100 Subject: [PATCH 002/107] feat: add new street project --- .../mapswipe_workers/definitions.py | 3 + .../project_types/__init__.py | 2 + .../project_types/street/__init__.py | 0 .../project_types/street/process_mapillary.py | 2 + .../project_types/street/project.py | 219 ++++++++++++++++++ .../unittests/test_project_type_street.py | 37 +++ 6 files changed, 263 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/__init__.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/project.py create mode 100644 mapswipe_workers/tests/unittests/test_project_type_street.py diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index bb8c7296f..2ddf22c0b 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -134,6 +134,7 @@ class ProjectType(Enum): COMPLETENESS = 4 MEDIA_CLASSIFICATION = 5 DIGITIZATION = 6 + STREET = 7 @property def constructor(self): @@ -145,6 +146,7 @@ def constructor(self): DigitizationProject, FootprintProject, MediaClassificationProject, + StreetProject, ) project_type_classes = { @@ -154,6 +156,7 @@ def constructor(self): 4: CompletenessProject, 5: MediaClassificationProject, 6: DigitizationProject, + 7: StreetProject, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index a07ff38be..048eda71c 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -8,6 +8,7 @@ from .tile_map_service.classification.tutorial import ClassificationTutorial from .tile_map_service.completeness.project import CompletenessProject from .tile_map_service.completeness.tutorial import CompletenessTutorial +from .street.project import StreetProject __all__ = [ "ClassificationProject", @@ -20,4 +21,5 @@ "FootprintProject", "FootprintTutorial", "DigitizationProject", + "StreetProject", ] diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py new file mode 100644 index 000000000..db96c3640 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py @@ -0,0 +1,2 @@ +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py new file mode 100644 index 000000000..502c1d36c --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -0,0 +1,219 @@ +import json +import os +import urllib +import math + +from osgeo import ogr +from dataclasses import asdict, dataclass +from mapswipe_workers.definitions import DATA_PATH, logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.firebase_to_postgres.transfer_results import ( + results_to_file, + save_results_to_postgres, + truncate_temp_results, +) +from mapswipe_workers.generate_stats.project_stats import ( + get_statistics_for_integer_result_project, +) +from mapswipe_workers.project_types.project import ( + BaseProject, BaseTask, BaseGroup +) +from mapswipe_workers.project_types.street.process_mapillary import get_image_ids +from mapswipe_workers.utils.api_calls import geojsonToFeatureCollection, ohsome + + +@dataclass +class StreetGroup(BaseGroup): + # todo: does client use this, or only for the implementation of project creation? + pass + + +@dataclass +class StreetTask(BaseTask): + pass + +class StreetProject(BaseProject): + def __init__(self, project_draft): + super().__init__(project_draft) + self.groups: Dict[str, MediaClassificationGroup] = {} + self.tasks: Dict[str, List[MediaClassificationTask]] = ( + {} + ) + + self.geometry = project_draft["geometry"] + self.imageList = get_image_ids() + + def save_tasks_to_firebase(self, projectId: str, tasks: dict): + firebase = Firebase() + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=True) + + @staticmethod + def results_to_postgres(results: dict, project_id: str, filter_mode: bool): + """How to move the result data from firebase to postgres.""" + results_file, user_group_results_file = results_to_file(results, project_id) + truncate_temp_results() + save_results_to_postgres(results_file, project_id, filter_mode) + return user_group_results_file + + @staticmethod + def get_per_project_statistics(project_id, project_info): + """How to aggregate the project results.""" + return get_statistics_for_integer_result_project( + project_id, project_info, generate_hot_tm_geometries=False + ) + + def validate_geometries(self): + raw_input_file = ( + f"{DATA_PATH}/input_geometries/raw_input_{self.projectId}.geojson" + ) + + if not os.path.isdir(f"{DATA_PATH}/input_geometries"): + os.mkdir(f"{DATA_PATH}/input_geometries") + + valid_input_file = ( + f"{DATA_PATH}/input_geometries/valid_input_{self.projectId}.geojson" + ) + + logger.info( + f"{self.projectId}" + f" - __init__ - " + f"downloaded input geometries from url and saved as file: " + f"{raw_input_file}" + ) + self.inputGeometries = raw_input_file + + # open the raw input file and get layer + driver = ogr.GetDriverByName("GeoJSON") + datasource = driver.Open(raw_input_file, 0) + try: + layer = datasource.GetLayer() + LayerDefn = layer.GetLayerDefn() + except AttributeError: + raise CustomError("Value error in input geometries file") + + # create layer for valid_input_file to store all valid geometries + outDriver = ogr.GetDriverByName("GeoJSON") + # Remove output geojson if it already exists + if os.path.exists(valid_input_file): + outDriver.DeleteDataSource(valid_input_file) + outDataSource = outDriver.CreateDataSource(valid_input_file) + outLayer = outDataSource.CreateLayer( + "geometries", geom_type=ogr.wkbMultiPolygon + ) + for i in range(0, LayerDefn.GetFieldCount()): + fieldDefn = LayerDefn.GetFieldDefn(i) + outLayer.CreateField(fieldDefn) + outLayerDefn = outLayer.GetLayerDefn() + + # check if raw_input_file layer is empty + feature_count = layer.GetFeatureCount() + if feature_count < 1: + err = "empty file. No geometries provided" + # TODO: How to user logger and exceptions? + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise CustomError(err) + elif feature_count > 100000: + err = f"Too many Geometries: {feature_count}" + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise CustomError(err) + + # get geometry as wkt + # get the bounding box/ extent of the layer + extent = layer.GetExtent() + # Create a Polygon from the extent tuple + ring = ogr.Geometry(ogr.wkbLinearRing) + ring.AddPoint(extent[0], extent[2]) + ring.AddPoint(extent[1], extent[2]) + ring.AddPoint(extent[1], extent[3]) + ring.AddPoint(extent[0], extent[3]) + ring.AddPoint(extent[0], extent[2]) + poly = ogr.Geometry(ogr.wkbPolygon) + poly.AddGeometry(ring) + wkt_geometry = poly.ExportToWkt() + + # check if the input geometry is a valid polygon + for feature in layer: + feat_geom = feature.GetGeometryRef() + geom_name = feat_geom.GetGeometryName() + fid = feature.GetFID() + + if not feat_geom.IsValid(): + layer.DeleteFeature(fid) + logger.warning( + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted invalid feature {fid}" + ) + + # we accept only POLYGON or MULTIPOLYGON geometries + elif geom_name not in ["POLYGON", "MULTIPOLYGON"]: + layer.DeleteFeature(fid) + logger.warning( + f"{self.projectId}" + f" - check_input_geometries - " + f"deleted non polygon feature {fid}" + ) + + else: + # Create output Feature + outFeature = ogr.Feature(outLayerDefn) + # Add field values from input Layer + for i in range(0, outLayerDefn.GetFieldCount()): + outFeature.SetField( + outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i) + ) + outFeature.SetGeometry(feat_geom) + outLayer.CreateFeature(outFeature) + outFeature = None + + # check if layer is empty + if layer.GetFeatureCount() < 1: + err = "no geometries left after checking validity and geometry type." + logger.warning(f"{self.projectId} - check_input_geometry - {err}") + raise Exception(err) + + del datasource + del outDataSource + del layer + + self.inputGeometriesFileName = valid_input_file + + logger.info( + f"{self.projectId}" + f" - check_input_geometry - " + f"filtered correct input geometries and created file: " + f"{valid_input_file}" + ) + return wkt_geometry + + def create_groups(self): + self.numberOfGroups = math.ceil(len(self.imageList) / self.groupSize) + for group_id in range(self.numberOfGroups): + self.groups[f"g{group_id}"] = StreetGroup( + projectId=self.projectId, + groupId=f"g{group_id}", + progress=0, + finishedCount=0, + requiredCount=0, + numberOfTasks=self.groupSize, + ) + + def create_tasks(self): + if len(self.groups) == 0: + raise ValueError("Groups needs to be created before tasks can be created.") + for group_id, group in self.groups.items(): + self.tasks[group_id] = [] + for i in range(self.groupSize): + task = StreetTask( + projectId=self.projectId, + groupId=group_id, + taskId=self.imageList.pop(), + ) + self.tasks[group_id].append(task) + + # list now empty? if usual group size is not reached + # the actual number of tasks for the group is updated + if not self.imageList: + group.numberOfTasks = i + 1 + break + diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py new file mode 100644 index 000000000..70c97b13d --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -0,0 +1,37 @@ +import json +import os +import unittest +from unittest.mock import patch + +from mapswipe_workers.project_types import StreetProject +from tests import fixtures + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self) -> None: + project_draft = fixtures.get_fixture( + os.path.join( + "projectDrafts", + "street.json", + ) + ) + project_draft["projectDraftId"] = "foo" + self.project = StreetProject(project_draft) + + def test_init(self): + self.assertEqual(self.project.geometry["type"], "FeatureCollection") + + def test_create_group(self): + self.project.create_groups() + self.assertTrue(self.project.groups) + + def test_create_tasks(self): + imageId = self.project.imageList[-1] + self.project.create_groups() + self.project.create_tasks() + self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) + #self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) + + +if __name__ == "__main__": + unittest.main() From 33bc5ece00a13d17558b656b99e4da2cfe7947a8 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 29 Oct 2024 17:10:14 +0100 Subject: [PATCH 003/107] feat: add tutorial to street project --- .../project_types/street/tutorial.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py new file mode 100644 index 000000000..cfbfc0ead --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -0,0 +1,14 @@ +from mapswipe_workers.project_types.tutorial import BaseTutorial + + +class StreetTutorial(BaseTutorial): + """The subclass for an TMS Grid based Tutorial.""" + + def save_tutorial(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_groups(self): + raise NotImplementedError("Currently Street has no Tutorial") + + def create_tutorial_tasks(self): + raise NotImplementedError("Currently Street has no Tutorial") From ec59ee0ff30584a4ec5dcd37732b42e03a46dc22 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:13:37 +0100 Subject: [PATCH 004/107] refactor: move process_mapillary.py to utils --- .../mapswipe_workers/project_types/street/process_mapillary.py | 2 -- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py create mode 100644 mapswipe_workers/mapswipe_workers/utils/process_mapillary.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py b/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py deleted file mode 100644 index db96c3640..000000000 --- a/mapswipe_workers/mapswipe_workers/project_types/street/process_mapillary.py +++ /dev/null @@ -1,2 +0,0 @@ -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py new file mode 100644 index 000000000..7287a65e9 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -0,0 +1,2 @@ +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] From 78f28df91e64298fda65ed1f625512b8b9b45b28 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:14:35 +0100 Subject: [PATCH 005/107] feat: add functions to process mapillary data --- .../utils/process_mapillary.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 7287a65e9..188781703 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -1,2 +1,126 @@ +import mercantile +import json +import requests +import os +import time +from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString +import pandas as pd +from vt2geojson import tools as vt2geojson_tools +from concurrent.futures import ThreadPoolExecutor, as_completed + def get_image_ids(): return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + +def create_tiles(polygon, level): + if not isinstance(polygon, (Polygon, MultiPolygon)): + return pd.DataFrame(columns=['x', 'y', 'z', 'geometry']) + if isinstance(polygon, Polygon): + polygon = MultiPolygon([polygon]) + + tiles = [] + for i, poly in enumerate(polygon.geoms): + tiles.extend(list(mercantile.tiles(*poly.bounds, level))) + + bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] + bbox_polygons = [box(*bbox) for bbox in bbox_list] + tiles = pd.DataFrame({ + 'x': [tile.x for tile in tiles], + 'y': [tile.y for tile in tiles], + 'z': [tile.z for tile in tiles], + 'geometry': bbox_polygons}) + + return tiles + + +def download_and_process_tile(row, token, attempt_limit=3): + z = row["z"] + x = row["x"] + y = row["y"] + endpoint = "mly1_computed_public" + url = f"https://tiles.mapillary.com/maps/vtp/{endpoint}/2/{z}/{x}/{y}?access_token={token}" + + attempt = 0 + while attempt < attempt_limit: + try: + r = requests.get(url) + assert r.status_code == 200, r.content + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) + + data = [] + for feature in features: + geometry = feature.get('geometry', {}) + properties = feature.get('properties', {}) + geometry_type = geometry.get('type', None) + coordinates = geometry.get('coordinates', []) + + element_geometry = None + if geometry_type == 'Point': + element_geometry = Point(coordinates) + elif geometry_type == 'LineString': + element_geometry = LineString(coordinates) + elif geometry_type == 'MultiLineString': + element_geometry = MultiLineString(coordinates) + elif geometry_type == 'Polygon': + element_geometry = Polygon(coordinates[0]) + elif geometry_type == 'MultiPolygon': + element_geometry = MultiPolygon(coordinates) + + # Append the dictionary with geometry and properties + row = {'geometry': element_geometry, **properties} + data.append(row) + + data = pd.DataFrame(data) + + if not data.empty: + return data, None + except Exception as e: + print(f"An exception occurred while requesting a tile: {e}") + attempt += 1 + + print(f"A tile could not be downloaded: {row}") + return None, row + + +def coordinate_download(polygon, level, token, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4): + tiles = create_tiles(polygon, level) + + downloaded_metadata = [] + failed_tiles = [] + + if not tiles.empty: + if not use_concurrency: + workers = 1 + + futures = [] + start_time = time.time() + with ThreadPoolExecutor(max_workers=workers) as executor: + for index, row in tiles.iterrows(): + futures.append(executor.submit(download_and_process_tile, row, token, attempt_limit)) + + for future in as_completed(futures): + if future is not None: + df, failed_row = future.result() + + if df is not None and not df.empty: + downloaded_metadata.append(df) + if failed_row is not None: + failed_tiles.append(failed_row) + + end_time = time.time() + total_time = end_time - start_time + + total_tiles = len(tiles) + average_time_per_tile = total_time / total_tiles if total_tiles > 0 else 0 + + print(f"Total time for downloading {total_tiles} tiles: {total_time:.2f} seconds") + print(f"Average time per tile: {average_time_per_tile:.2f} seconds") + + if len(downloaded_metadata): + downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) + else: + downloaded_metadata = pd.DataFrame(downloaded_metadata) + + failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) + + return downloaded_metadata, failed_tiles From 3a7ac21fbf48edfdc29f84b600eedc7177127c93 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 11:15:10 +0100 Subject: [PATCH 006/107] refactor: removed unused imports and classes --- .../mapswipe_workers/project_types/street/project.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 502c1d36c..5a2f834af 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -4,7 +4,7 @@ import math from osgeo import ogr -from dataclasses import asdict, dataclass +from dataclasses import dataclass from mapswipe_workers.definitions import DATA_PATH, logger from mapswipe_workers.firebase.firebase import Firebase from mapswipe_workers.firebase_to_postgres.transfer_results import ( @@ -18,8 +18,7 @@ from mapswipe_workers.project_types.project import ( BaseProject, BaseTask, BaseGroup ) -from mapswipe_workers.project_types.street.process_mapillary import get_image_ids -from mapswipe_workers.utils.api_calls import geojsonToFeatureCollection, ohsome +from mapswipe_workers.utils.process_mapillary import get_image_ids @dataclass @@ -35,8 +34,8 @@ class StreetTask(BaseTask): class StreetProject(BaseProject): def __init__(self, project_draft): super().__init__(project_draft) - self.groups: Dict[str, MediaClassificationGroup] = {} - self.tasks: Dict[str, List[MediaClassificationTask]] = ( + self.groups: Dict[str, StreetGroup] = {} + self.tasks: Dict[str, List[StreetTask]] = ( {} ) @@ -45,7 +44,7 @@ def __init__(self, project_draft): def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() - firebase.save_tasks_to_firebase(projectId, tasks, useCompression=True) + firebase.save_tasks_to_firebase(projectId, tasks) @staticmethod def results_to_postgres(results: dict, project_id: str, filter_mode: bool): From 8e5c55c879516db9a42e69dad799832540646f9d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 14:34:28 +0100 Subject: [PATCH 007/107] feat: add function to convert input geojson to polygon --- .../utils/process_mapillary.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 188781703..33dc537c8 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -3,13 +3,13 @@ import requests import os import time -from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString +from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString, unary_union +from shapely.geometry import shape import pandas as pd from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +import logging def create_tiles(polygon, level): @@ -46,7 +46,6 @@ def download_and_process_tile(row, token, attempt_limit=3): r = requests.get(url) assert r.status_code == 200, r.content features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) - data = [] for feature in features: geometry = feature.get('geometry', {}) @@ -124,3 +123,27 @@ def coordinate_download(polygon, level, token, use_concurrency=True, attempt_lim failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) return downloaded_metadata, failed_tiles + +def geojson_to_polygon(geojson_data): + if geojson_data["type"] == "FeatureCollection": + features = geojson_data["features"] + elif geojson_data["type"] == "Feature": + features = [geojson_data] + else: + raise ValueError("Unsupported GeoJSON type.") + + polygons = [] + for feature in features: + geometry = shape(feature["geometry"]) + if isinstance(geometry, (Polygon, MultiPolygon)): + polygons.append(geometry) + else: + raise ValueError("Non-polygon geometries cannot be combined into a MultiPolygon.") + + combined_multipolygon = unary_union(polygons) + + return combined_multipolygon + + +def get_image_ids(): + return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file From dd4e220dcee6b41b47d26e7546df37a5b55565e0 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 14:35:07 +0100 Subject: [PATCH 008/107] tests: add unittests for process_mapillary --- .../tests/unittests/test_process_mapillary.py | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 mapswipe_workers/tests/unittests/test_process_mapillary.py diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py new file mode 100644 index 000000000..a9d102e49 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -0,0 +1,178 @@ +import unittest +import os +import json +from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection +import pandas as pd +from unittest.mock import patch, MagicMock +from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon + + +# Assuming create_tiles, download_and_process_tile, and coordinate_download are imported + +class TestTileGroupingFunctions(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "feature_collection.json", + ), + "r", + ) as file: + cls.fixture_data = json.load(file) + + def setUp(self): + self.token = "test_token" + self.level = 14 + self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + self.test_multipolygon = MultiPolygon([self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])]) + self.empty_polygon = Polygon() + self.empty_geometry = GeometryCollection() + + def test_create_tiles_with_valid_polygon(self): + tiles = create_tiles(self.test_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_multipolygon(self): + tiles = create_tiles(self.test_multipolygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_empty_polygon(self): + tiles = create_tiles(self.empty_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_create_tiles_with_empty_geometry(self): + tiles = create_tiles(self.empty_geometry, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]]}}, + {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]]}} + ] + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_single_feature_polygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]] + } + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, Polygon) + + def test_geojson_to_polygon_single_feature_multipolygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]] + ] + } + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}} + ] + } + with self.assertRaises(ValueError) as context: + geojson_to_polygon(geojson_data) + self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") + + def test_geojson_to_polygon_empty_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [] + } + result = geojson_to_polygon(geojson_data) + self.assertTrue(result.is_empty) + + def test_geojson_to_polygon_contribution_geojson(self): + result = geojson_to_polygon(self.fixture_data) + self.assertIsInstance(result, Polygon) + + @patch('mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson') + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): + # Mock the response from requests.get + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = b'mock vector tile data' # Example mock data + mock_get.return_value = mock_response + + # Mock the return value of vt_bytes_to_geojson + mock_vt2geojson.return_value = { + "features": [ + {"geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": {"id": 1}} + ] + } + + row = {'x': 1, 'y': 1, 'z': 14} + token = 'test_token' + + result, failed = download_and_process_tile(row, token) + + # Assertions + self.assertIsNone(failed) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') + + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_failure(self, mock_get): + # Mock a failed response + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response + + row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + result, failed = download_and_process_tile(row, self.token) + + self.assertIsNone(result) + self.assertIsNotNone(failed) + + @patch('mapswipe_workers.utils.process_mapillary') + def test_coordinate_download(self, mock_download_and_process_tile): + + mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + + self.assertIsInstance(metadata, pd.DataFrame) + self.assertTrue(failed.empty) + + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download_with_failures(self, mock_download_and_process_tile): + # Simulate failed tile download + mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + + self.assertTrue(metadata.empty) + self.assertFalse(failed.empty) + + +if __name__ == '__main__': + unittest.main() From b274af3d917d539ad41d007d439f230cbc0c563e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:16:50 +0100 Subject: [PATCH 009/107] fix: wrong indention and patch in test_process_mapillary --- .../tests/unittests/test_process_mapillary.py | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index a9d102e49..abca8657c 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -24,6 +24,17 @@ def setUpClass(cls): ) as file: cls.fixture_data = json.load(file) + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + cls.fixture_df = pd.read_csv(file) + def setUp(self): self.token = "test_token" self.level = 14 @@ -140,38 +151,36 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): self.assertEqual(len(result), 1) self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') - @patch('mapswipe_workers.utils.process_mapillary.requests.get') - def test_download_and_process_tile_failure(self, mock_get): - # Mock a failed response - mock_response = MagicMock() - mock_response.status_code = 500 - mock_get.return_value = mock_response - - row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) - result, failed = download_and_process_tile(row, self.token) + @patch('mapswipe_workers.utils.process_mapillary.requests.get') + def test_download_and_process_tile_failure(self, mock_get): + # Mock a failed response + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response - self.assertIsNone(result) - self.assertIsNotNone(failed) + row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + result, failed = download_and_process_tile(row, self.token) - @patch('mapswipe_workers.utils.process_mapillary') - def test_coordinate_download(self, mock_download_and_process_tile): + self.assertIsNone(result) + self.assertIsNotNone(failed) - mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) - self.assertIsInstance(metadata, pd.DataFrame) - self.assertTrue(failed.empty) + self.assertIsInstance(metadata, pd.DataFrame) + self.assertTrue(failed.empty) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') - def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - # Simulate failed tile download - mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + def test_coordinate_download_with_failures(self, mock_download_and_process_tile): + mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) - self.assertTrue(metadata.empty) - self.assertFalse(failed.empty) + self.assertTrue(metadata.empty) + self.assertFalse(failed.empty) if __name__ == '__main__': From 7ec92ee774bade0e8cff74e32ef459c60cacb5b2 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:41:45 +0100 Subject: [PATCH 010/107] feat: add filtering and handling of mapillary response --- example.env | 3 ++ .../utils/process_mapillary.py | 31 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/example.env b/example.env index f4e8c46a1..e3b856ea2 100644 --- a/example.env +++ b/example.env @@ -75,3 +75,6 @@ COMMUNITY_DASHBOARD_GRAPHQL_ENDPOINT=https://api.example.com/graphql/ COMMUNITY_DASHBOARD_SENTRY_DSN= COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org + +# Mapillary +MAPILLARY_ACCESS_TOKEN= \ No newline at end of file diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 33dc537c8..2d51a297b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -144,6 +144,31 @@ def geojson_to_polygon(geojson_data): return combined_multipolygon - -def get_image_ids(): - return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] \ No newline at end of file +def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): + df['captured_at'] = pd.to_datetime(df['captured_at'], unit='ms') + start_time = pd.Timestamp(start_time) + + if end_time is None: + end_time = pd.Timestamp.now() + + filtered_df = df[(df['captured_at'] >= start_time) & (df['captured_at'] <= end_time)] + return filtered_df + +def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + df = results_df.copy() + if is_pano is not None: + df = df[df["is_pano"] == is_pano] + if organization_id is not None: + df = df[df["organization_id"] == organization_id] + if start_time is not None: + df = filter_by_timerange(df, start_time, end_time) + + return df + + +def get_image_ids(aoi_geojson, level = 14, attempt_limit = 3, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + aoi_polygon = geojson_to_polygon(aoi_geojson) + token = os.getenv("MAPILLARY_ACCESS_TOKEN") + downloaded_metadata, failed_tiles = coordinate_download(aoi_polygon, level, token, attempt_limit) + downloaded_metadata = filter_results(downloaded_metadata, is_pano, organization_id, start_time, end_time) + return downloaded_metadata["image_id"].tolist() \ No newline at end of file From 97e552c907ce765600e00fc565c891951e79c6be Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 30 Oct 2024 15:42:05 +0100 Subject: [PATCH 011/107] tests: add tests for filtering and handling of mapillary response --- .../tests/fixtures/mapillary_response.csv | 8 ++++ .../tests/unittests/test_process_mapillary.py | 45 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 mapswipe_workers/tests/fixtures/mapillary_response.csv diff --git a/mapswipe_workers/tests/fixtures/mapillary_response.csv b/mapswipe_workers/tests/fixtures/mapillary_response.csv new file mode 100644 index 000000000..2988a1dc4 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_response.csv @@ -0,0 +1,8 @@ +geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +POINT (38.995129466056824 -6.785243670271996),1453463352000,102506575322825,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,1 +POINT (38.99839103221893 -6.7866606090858),1679465543298,118200124520512,603013591724120,,False,104.904435758052,Tj9u08PcRnQEAU13yeNhkr +POINT (39.000306129455566 -6.787576822940906),1679465564715,118200124520512,547943597327117,,False,103.540136343809,Tj9u08PcRnQEAU13yeNhkr +POINT (38.9906769990921 -6.783315348346505),1453463400000,102506575322825,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ +POINT (38.99797797203064 -6.786490150501777),1679465534081,118200124520512,3271824289814895,,False,112.637054443359,Tj9u08PcRnQEAU13yeNhkr +POINT (39.00127172470093 -6.787981661065601),1453463294000,102506575322825,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ + diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index abca8657c..7e9413c6a 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -4,7 +4,7 @@ from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection import pandas as pd from unittest.mock import patch, MagicMock -from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon +from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon, filter_by_timerange, filter_results # Assuming create_tiles, download_and_process_tile, and coordinate_download are imported @@ -182,6 +182,49 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) self.assertTrue(metadata.empty) self.assertFalse(failed.empty) + def test_filter_within_time_range(self): + start_time = '2016-01-20 00:00:00' + end_time = '2022-01-21 23:59:59' + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df['captured_at'] <= pd.to_datetime(end_time))) + + def test_filter_without_end_time(self): + start_time = '2020-01-20 00:00:00' + filtered_df = filter_by_timerange(self.fixture_df, start_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + + def test_filter_time_no_data(self): + start_time = '2016-01-30 00:00:00' + end_time = '2016-01-31 00:00:00' + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + self.assertTrue(filtered_df.empty) + + def test_filter_default(self): + filtered_df = filter_results(self.fixture_df) + self.assertTrue(len(filtered_df) == len(self.fixture_df)) + + def test_filter_pano_true(self): + filtered_df = filter_results(self.fixture_df, is_pano=True) + self.assertEqual(len(filtered_df), 3) + + def test_filter_pano_false(self): + filtered_df = filter_results(self.fixture_df, is_pano=False) + self.assertEqual(len(filtered_df), 3) + + def test_filter_organization_id(self): + filtered_df = filter_results(self.fixture_df, organization_id=1) + self.assertEqual(len(filtered_df), 1) + + def test_filter_time_range(self): + start_time = '2016-01-20 00:00:00' + end_time = '2022-01-21 23:59:59' + filtered_df = filter_results(self.fixture_df, start_time=start_time, end_time=end_time) + self.assertEqual(len(filtered_df), 3) if __name__ == '__main__': unittest.main() From 53dfda5621aba90cdb08ca89d288eb738d4d8b5f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:04:42 +0100 Subject: [PATCH 012/107] refactor: use mapillary incredentials in variables --- mapswipe_workers/mapswipe_workers/definitions.py | 2 ++ .../mapswipe_workers/utils/process_mapillary.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 2ddf22c0b..c9dec6d79 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -16,6 +16,8 @@ OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] +MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 2d51a297b..5e1d00c47 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -8,8 +8,7 @@ import pandas as pd from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed - -import logging +from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY def create_tiles(polygon, level): @@ -33,12 +32,11 @@ def create_tiles(polygon, level): return tiles -def download_and_process_tile(row, token, attempt_limit=3): +def download_and_process_tile(row, attempt_limit=3): z = row["z"] x = row["x"] y = row["y"] - endpoint = "mly1_computed_public" - url = f"https://tiles.mapillary.com/maps/vtp/{endpoint}/2/{z}/{x}/{y}?access_token={token}" + url = f"{MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={MAPILLARY_API_KEY}" attempt = 0 while attempt < attempt_limit: @@ -81,7 +79,9 @@ def download_and_process_tile(row, token, attempt_limit=3): return None, row -def coordinate_download(polygon, level, token, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4): +def coordinate_download( + polygon, level, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4 +): tiles = create_tiles(polygon, level) downloaded_metadata = [] @@ -95,7 +95,9 @@ def coordinate_download(polygon, level, token, use_concurrency=True, attempt_lim start_time = time.time() with ThreadPoolExecutor(max_workers=workers) as executor: for index, row in tiles.iterrows(): - futures.append(executor.submit(download_and_process_tile, row, token, attempt_limit)) + futures.append( + executor.submit(download_and_process_tile, row, attempt_limit) + ) for future in as_completed(futures): if future is not None: From 481030c289cc99ce0fb7a6b9e638b05100bb03ab Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:06:19 +0100 Subject: [PATCH 013/107] refactor: reformat file process_mapillary.py --- .../utils/process_mapillary.py | 108 ++++++++++++------ 1 file changed, 70 insertions(+), 38 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5e1d00c47..4f1a55084 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -3,7 +3,15 @@ import requests import os import time -from shapely import box, Polygon, MultiPolygon, Point, LineString, MultiLineString, unary_union +from shapely import ( + box, + Polygon, + MultiPolygon, + Point, + LineString, + MultiLineString, + unary_union, +) from shapely.geometry import shape import pandas as pd from vt2geojson import tools as vt2geojson_tools @@ -13,7 +21,7 @@ def create_tiles(polygon, level): if not isinstance(polygon, (Polygon, MultiPolygon)): - return pd.DataFrame(columns=['x', 'y', 'z', 'geometry']) + return pd.DataFrame(columns=["x", "y", "z", "geometry"]) if isinstance(polygon, Polygon): polygon = MultiPolygon([polygon]) @@ -23,11 +31,14 @@ def create_tiles(polygon, level): bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] bbox_polygons = [box(*bbox) for bbox in bbox_list] - tiles = pd.DataFrame({ - 'x': [tile.x for tile in tiles], - 'y': [tile.y for tile in tiles], - 'z': [tile.z for tile in tiles], - 'geometry': bbox_polygons}) + tiles = pd.DataFrame( + { + "x": [tile.x for tile in tiles], + "y": [tile.y for tile in tiles], + "z": [tile.z for tile in tiles], + "geometry": bbox_polygons, + } + ) return tiles @@ -43,28 +54,30 @@ def download_and_process_tile(row, attempt_limit=3): try: r = requests.get(url) assert r.status_code == 200, r.content - features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get('features', []) + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( + "features", [] + ) data = [] for feature in features: - geometry = feature.get('geometry', {}) - properties = feature.get('properties', {}) - geometry_type = geometry.get('type', None) - coordinates = geometry.get('coordinates', []) + geometry = feature.get("geometry", {}) + properties = feature.get("properties", {}) + geometry_type = geometry.get("type", None) + coordinates = geometry.get("coordinates", []) element_geometry = None - if geometry_type == 'Point': + if geometry_type == "Point": element_geometry = Point(coordinates) - elif geometry_type == 'LineString': + elif geometry_type == "LineString": element_geometry = LineString(coordinates) - elif geometry_type == 'MultiLineString': + elif geometry_type == "MultiLineString": element_geometry = MultiLineString(coordinates) - elif geometry_type == 'Polygon': + elif geometry_type == "Polygon": element_geometry = Polygon(coordinates[0]) - elif geometry_type == 'MultiPolygon': + elif geometry_type == "MultiPolygon": element_geometry = MultiPolygon(coordinates) # Append the dictionary with geometry and properties - row = {'geometry': element_geometry, **properties} + row = {"geometry": element_geometry, **properties} data.append(row) data = pd.DataFrame(data) @@ -92,7 +105,6 @@ def coordinate_download( workers = 1 futures = [] - start_time = time.time() with ThreadPoolExecutor(max_workers=workers) as executor: for index, row in tiles.iterrows(): futures.append( @@ -108,24 +120,18 @@ def coordinate_download( if failed_row is not None: failed_tiles.append(failed_row) - end_time = time.time() - total_time = end_time - start_time - - total_tiles = len(tiles) - average_time_per_tile = total_time / total_tiles if total_tiles > 0 else 0 - - print(f"Total time for downloading {total_tiles} tiles: {total_time:.2f} seconds") - print(f"Average time per tile: {average_time_per_tile:.2f} seconds") - if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: downloaded_metadata = pd.DataFrame(downloaded_metadata) - failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(drop=True) + failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( + drop=True + ) return downloaded_metadata, failed_tiles + def geojson_to_polygon(geojson_data): if geojson_data["type"] == "FeatureCollection": features = geojson_data["features"] @@ -140,23 +146,35 @@ def geojson_to_polygon(geojson_data): if isinstance(geometry, (Polygon, MultiPolygon)): polygons.append(geometry) else: - raise ValueError("Non-polygon geometries cannot be combined into a MultiPolygon.") + raise ValueError( + "Non-polygon geometries cannot be combined into a MultiPolygon." + ) combined_multipolygon = unary_union(polygons) return combined_multipolygon + def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): - df['captured_at'] = pd.to_datetime(df['captured_at'], unit='ms') + df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") start_time = pd.Timestamp(start_time) if end_time is None: end_time = pd.Timestamp.now() - filtered_df = df[(df['captured_at'] >= start_time) & (df['captured_at'] <= end_time)] + filtered_df = df[ + (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) + ] return filtered_df -def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): + +def filter_results( + results_df: pd.DataFrame, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): df = results_df.copy() if is_pano is not None: df = df[df["is_pano"] == is_pano] @@ -168,9 +186,23 @@ def filter_results(results_df: pd.DataFrame, is_pano: bool = None, organization_ return df -def get_image_ids(aoi_geojson, level = 14, attempt_limit = 3, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None): +def get_image_metadata( + aoi_geojson, + level=14, + attempt_limit=3, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): aoi_polygon = geojson_to_polygon(aoi_geojson) - token = os.getenv("MAPILLARY_ACCESS_TOKEN") - downloaded_metadata, failed_tiles = coordinate_download(aoi_polygon, level, token, attempt_limit) - downloaded_metadata = filter_results(downloaded_metadata, is_pano, organization_id, start_time, end_time) - return downloaded_metadata["image_id"].tolist() \ No newline at end of file + downloaded_metadata, failed_tiles = coordinate_download( + aoi_polygon, level, attempt_limit + ) + downloaded_metadata = filter_results( + downloaded_metadata, is_pano, organization_id, start_time, end_time + ) + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } From 1187561e96149c1cc92a9410f47c50e3c6d6e498 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:11:36 +0100 Subject: [PATCH 014/107] feat: add geometry to StreetTask and refactoring --- .../project_types/street/project.py | 157 ++++-------------- 1 file changed, 33 insertions(+), 124 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 5a2f834af..3dd3451d5 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -15,10 +15,16 @@ from mapswipe_workers.generate_stats.project_stats import ( get_statistics_for_integer_result_project, ) -from mapswipe_workers.project_types.project import ( - BaseProject, BaseTask, BaseGroup +from mapswipe_workers.utils.validate_input import ( + check_if_layer_is_empty, + load_geojson_to_ogr, + build_multipolygon_from_layer_geometries, + check_if_layer_has_too_many_geometries, + save_geojson_to_file, + multipolygon_to_wkt ) -from mapswipe_workers.utils.process_mapillary import get_image_ids +from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup +from mapswipe_workers.utils.process_mapillary import get_image_metadata @dataclass @@ -29,18 +35,22 @@ class StreetGroup(BaseGroup): @dataclass class StreetTask(BaseTask): - pass + geometry: str + data: str + class StreetProject(BaseProject): def __init__(self, project_draft): super().__init__(project_draft) self.groups: Dict[str, StreetGroup] = {} - self.tasks: Dict[str, List[StreetTask]] = ( - {} - ) + self.tasks: Dict[str, List[StreetTask]] = {} self.geometry = project_draft["geometry"] - self.imageList = get_image_ids() + ImageMetadata = get_image_metadata(self.geometry) + + + self.imageIds = ImageMetadata["ids"] + self.imageGeometries = ImageMetadata["geometries"] def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() @@ -62,131 +72,29 @@ def get_per_project_statistics(project_id, project_info): ) def validate_geometries(self): - raw_input_file = ( - f"{DATA_PATH}/input_geometries/raw_input_{self.projectId}.geojson" + self.inputGeometriesFileName = save_geojson_to_file( + self.projectId, self.geometry ) + layer, datasource = load_geojson_to_ogr(self.projectId, self.inputGeometriesFileName) - if not os.path.isdir(f"{DATA_PATH}/input_geometries"): - os.mkdir(f"{DATA_PATH}/input_geometries") - - valid_input_file = ( - f"{DATA_PATH}/input_geometries/valid_input_{self.projectId}.geojson" - ) + # check if inputs fit constraints + check_if_layer_is_empty(self.projectId, layer) - logger.info( - f"{self.projectId}" - f" - __init__ - " - f"downloaded input geometries from url and saved as file: " - f"{raw_input_file}" + multi_polygon, project_area = build_multipolygon_from_layer_geometries( + self.projectId, layer ) - self.inputGeometries = raw_input_file - - # open the raw input file and get layer - driver = ogr.GetDriverByName("GeoJSON") - datasource = driver.Open(raw_input_file, 0) - try: - layer = datasource.GetLayer() - LayerDefn = layer.GetLayerDefn() - except AttributeError: - raise CustomError("Value error in input geometries file") - - # create layer for valid_input_file to store all valid geometries - outDriver = ogr.GetDriverByName("GeoJSON") - # Remove output geojson if it already exists - if os.path.exists(valid_input_file): - outDriver.DeleteDataSource(valid_input_file) - outDataSource = outDriver.CreateDataSource(valid_input_file) - outLayer = outDataSource.CreateLayer( - "geometries", geom_type=ogr.wkbMultiPolygon - ) - for i in range(0, LayerDefn.GetFieldCount()): - fieldDefn = LayerDefn.GetFieldDefn(i) - outLayer.CreateField(fieldDefn) - outLayerDefn = outLayer.GetLayerDefn() - - # check if raw_input_file layer is empty - feature_count = layer.GetFeatureCount() - if feature_count < 1: - err = "empty file. No geometries provided" - # TODO: How to user logger and exceptions? - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise CustomError(err) - elif feature_count > 100000: - err = f"Too many Geometries: {feature_count}" - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise CustomError(err) - - # get geometry as wkt - # get the bounding box/ extent of the layer - extent = layer.GetExtent() - # Create a Polygon from the extent tuple - ring = ogr.Geometry(ogr.wkbLinearRing) - ring.AddPoint(extent[0], extent[2]) - ring.AddPoint(extent[1], extent[2]) - ring.AddPoint(extent[1], extent[3]) - ring.AddPoint(extent[0], extent[3]) - ring.AddPoint(extent[0], extent[2]) - poly = ogr.Geometry(ogr.wkbPolygon) - poly.AddGeometry(ring) - wkt_geometry = poly.ExportToWkt() - - # check if the input geometry is a valid polygon - for feature in layer: - feat_geom = feature.GetGeometryRef() - geom_name = feat_geom.GetGeometryName() - fid = feature.GetFID() - - if not feat_geom.IsValid(): - layer.DeleteFeature(fid) - logger.warning( - f"{self.projectId}" - f" - check_input_geometries - " - f"deleted invalid feature {fid}" - ) - - # we accept only POLYGON or MULTIPOLYGON geometries - elif geom_name not in ["POLYGON", "MULTIPOLYGON"]: - layer.DeleteFeature(fid) - logger.warning( - f"{self.projectId}" - f" - check_input_geometries - " - f"deleted non polygon feature {fid}" - ) - else: - # Create output Feature - outFeature = ogr.Feature(outLayerDefn) - # Add field values from input Layer - for i in range(0, outLayerDefn.GetFieldCount()): - outFeature.SetField( - outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i) - ) - outFeature.SetGeometry(feat_geom) - outLayer.CreateFeature(outFeature) - outFeature = None - - # check if layer is empty - if layer.GetFeatureCount() < 1: - err = "no geometries left after checking validity and geometry type." - logger.warning(f"{self.projectId} - check_input_geometry - {err}") - raise Exception(err) + check_if_layer_has_too_many_geometries(self.projectId, multi_polygon) del datasource - del outDataSource del layer - self.inputGeometriesFileName = valid_input_file - - logger.info( - f"{self.projectId}" - f" - check_input_geometry - " - f"filtered correct input geometries and created file: " - f"{valid_input_file}" - ) + logger.info(f"{self.projectId}" f" - validate geometry - " f"input geometry is correct.") + wkt_geometry = multipolygon_to_wkt(multi_polygon) return wkt_geometry def create_groups(self): - self.numberOfGroups = math.ceil(len(self.imageList) / self.groupSize) + self.numberOfGroups = math.ceil(len(self.imageIds) / self.groupSize) for group_id in range(self.numberOfGroups): self.groups[f"g{group_id}"] = StreetGroup( projectId=self.projectId, @@ -206,13 +114,14 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, - taskId=self.imageList.pop(), + data=str(self.imageGeometries.pop()), + geometry="", + taskId=self.imageIds.pop(), ) self.tasks[group_id].append(task) # list now empty? if usual group size is not reached # the actual number of tasks for the group is updated - if not self.imageList: + if not self.imageIds: group.numberOfTasks = i + 1 break - From 0bef2e01766f6d3b3534b2d6271236ad24f60117 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:13:24 +0100 Subject: [PATCH 015/107] tests(street): use simpler geometry and add filter parameters to testing json --- .../tests/fixtures/projectDrafts/street.json | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 6bc291e33..341756584 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -5,8 +5,34 @@ "features": [ { "type": "Feature", - "properties": {}, - "geometry": { "type": "Polygon", "coordinates": [ [ [ 39.0085, -6.7811 ], [ 39.0214, -6.7908 ], [ 39.0374, -6.7918 ], [ 39.0537, -6.7968 ], [ 39.0628, -6.8525 ], [ 39.0909, -6.8757 ], [ 39.1018, -6.8747 ], [ 39.1161, -6.8781 ], [ 39.1088, -6.9151 ], [ 39.089, -6.9307 ], [ 39.0864, -6.94 ], [ 39.0596, -6.9731 ], [ 39.0716, -6.9835 ], [ 39.0499, -7.0279 ], [ 39.0236, -7.0322 ], [ 39.038, -7.0499 ], [ 39.0693, -7.0328 ], [ 39.0826, -7.0671 ], [ 39.112, -7.0728 ], [ 39.1154, -7.0921 ], [ 39.1543, -7.0183 ], [ 39.1746, -7.0101 ], [ 39.1699, -6.9971 ], [ 39.196, -6.9848 ], [ 39.2163, -6.9934 ], [ 39.2502, -6.9509 ], [ 39.2768, -6.9486 ], [ 39.2881, -6.9529 ], [ 39.3136, -6.9595 ], [ 39.3276, -6.9483 ], [ 39.3686, -6.9726 ], [ 39.3826, -6.9677 ], [ 39.393, -7.0 ], [ 39.4038, -7.0284 ], [ 39.4037, -7.0379 ], [ 39.4028, -7.0741 ], [ 39.4, -7.0914 ], [ 39.4154, -7.1286 ], [ 39.4049, -7.1595 ], [ 39.4124, -7.1661 ], [ 39.4287, -7.1546 ], [ 39.4395, -7.1618 ], [ 39.4501, -7.1827 ], [ 39.4897, -7.1692 ], [ 39.51, -7.1833 ], [ 39.5378, -7.1229 ], [ 39.5529, -7.1335 ], [ 39.5453, -7.0318 ], [ 39.5376, -6.9873 ], [ 39.4976, -6.9093 ], [ 39.4717, -6.8731 ], [ 39.3557, -6.8455 ], [ 39.3205, -6.8175 ], [ 39.3074, -6.8193 ], [ 39.3002, -6.8253 ], [ 39.293, -6.8317 ], [ 39.2944, -6.818 ], [ 39.2963, -6.8084 ], [ 39.2855, -6.7859 ], [ 39.2808, -6.7347 ], [ 39.2609, -6.7602 ], [ 39.2344, -6.715 ], [ 39.2004, -6.6376 ], [ 39.165, -6.6022 ], [ 39.1353, -6.5748 ], [ 39.1215, -6.5662 ], [ 39.0945, -6.5987 ], [ 39.086, -6.5956 ], [ 39.0803, -6.6062 ], [ 39.0578, -6.7139 ], [ 39.059, -6.723 ], [ 39.0085, -6.7811 ] ], [ [ 39.301, -6.8372 ], [ 39.3048, -6.8555 ], [ 39.2967, -6.8469 ], [ 39.301, -6.8372 ] ], [ [ 39.3048, -6.8555 ], [ 39.2949, -6.8707 ], [ 39.2921, -6.8607 ], [ 39.3048, -6.8555 ] ] ] } + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } } ] }, @@ -19,5 +45,9 @@ "projectType": 7, "requestingOrganisation": "test", "verificationNumber": 3, - "groupSize": 25 + "groupSize": 25, + "isPano": false, + "startTimestamp": "2019-07-01T00:00:00.000Z", + "endTimestamp": null, + "organisationId": "1" } From 789b02769a4163e6519b2d91dc75feae05064986 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:18:27 +0100 Subject: [PATCH 016/107] refactor: tests for mapillary processing and street project --- .../tests/unittests/test_process_mapillary.py | 136 ++++++++++++------ .../unittests/test_project_type_street.py | 4 +- 2 files changed, 93 insertions(+), 47 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 7e9413c6a..18d9e3401 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -1,16 +1,30 @@ import unittest import os import json -from shapely.geometry import Polygon, MultiPolygon, Point, LineString, MultiLineString, GeometryCollection +from shapely.geometry import ( + Polygon, + MultiPolygon, + Point, + LineString, + MultiLineString, + GeometryCollection, +) import pandas as pd from unittest.mock import patch, MagicMock -from mapswipe_workers.utils.process_mapillary import create_tiles, download_and_process_tile, coordinate_download, geojson_to_polygon, filter_by_timerange, filter_results +from mapswipe_workers.utils.process_mapillary import ( + create_tiles, + download_and_process_tile, + coordinate_download, + geojson_to_polygon, + filter_by_timerange, + filter_results, +) # Assuming create_tiles, download_and_process_tile, and coordinate_download are imported -class TestTileGroupingFunctions(unittest.TestCase): +class TestTileGroupingFunctions(unittest.TestCase): @classmethod def setUpClass(cls): with open( @@ -39,7 +53,9 @@ def setUp(self): self.token = "test_token" self.level = 14 self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) - self.test_multipolygon = MultiPolygon([self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])]) + self.test_multipolygon = MultiPolygon( + [self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])] + ) self.empty_polygon = Polygon() self.empty_geometry = GeometryCollection() @@ -67,9 +83,21 @@ def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): geojson_data = { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]]}}, - {"type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]]}} - ] + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + }, + }, + ], } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, MultiPolygon) @@ -80,8 +108,8 @@ def test_geojson_to_polygon_single_feature_polygon(self): "type": "Feature", "geometry": { "type": "Polygon", - "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]] - } + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, Polygon) @@ -93,9 +121,9 @@ def test_geojson_to_polygon_single_feature_multipolygon(self): "type": "MultiPolygon", "coordinates": [ [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], - [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]] - ] - } + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + ], + }, } result = geojson_to_polygon(geojson_data) self.assertIsInstance(result, MultiPolygon) @@ -105,18 +133,18 @@ def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): geojson_data = { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}} - ] + { + "type": "Feature", + "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}, + } + ], } with self.assertRaises(ValueError) as context: geojson_to_polygon(geojson_data) self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") def test_geojson_to_polygon_empty_feature_collection(self): - geojson_data = { - "type": "FeatureCollection", - "features": [] - } + geojson_data = {"type": "FeatureCollection", "features": []} result = geojson_to_polygon(geojson_data) self.assertTrue(result.is_empty) @@ -124,24 +152,29 @@ def test_geojson_to_polygon_contribution_geojson(self): result = geojson_to_polygon(self.fixture_data) self.assertIsInstance(result, Polygon) - @patch('mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson') - @patch('mapswipe_workers.utils.process_mapillary.requests.get') + @patch( + "mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson" + ) + @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): # Mock the response from requests.get mock_response = MagicMock() mock_response.status_code = 200 - mock_response.content = b'mock vector tile data' # Example mock data + mock_response.content = b"mock vector tile data" # Example mock data mock_get.return_value = mock_response # Mock the return value of vt_bytes_to_geojson mock_vt2geojson.return_value = { "features": [ - {"geometry": {"type": "Point", "coordinates": [0, 0]}, "properties": {"id": 1}} + { + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": {"id": 1}, + } ] } - row = {'x': 1, 'y': 1, 'z': 14} - token = 'test_token' + row = {"x": 1, "y": 1, "z": 14} + token = "test_token" result, failed = download_and_process_tile(row, token) @@ -149,58 +182,68 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) - self.assertEqual(result['geometry'][0].wkt, 'POINT (0 0)') + self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") - @patch('mapswipe_workers.utils.process_mapillary.requests.get') + @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_failure(self, mock_get): # Mock a failed response mock_response = MagicMock() mock_response.status_code = 500 mock_get.return_value = mock_response - row = pd.Series({'x': 1, 'y': 1, 'z': self.level}) + row = pd.Series({"x": 1, "y": 1, "z": self.level}) result, failed = download_and_process_tile(row, self.token) self.assertIsNone(result) self.assertIsNotNone(failed) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = (pd.DataFrame([{"geometry": None}]), None) + mock_download_and_process_tile.return_value = ( + pd.DataFrame([{"geometry": None}]), + None, + ) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download( + self.test_polygon, self.level, self.token + ) self.assertIsInstance(metadata, pd.DataFrame) self.assertTrue(failed.empty) - @patch('mapswipe_workers.utils.process_mapillary.download_and_process_tile') + @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = (None, pd.Series({"x": 1, "y": 1, "z": self.level})) + mock_download_and_process_tile.return_value = ( + None, + pd.Series({"x": 1, "y": 1, "z": self.level}), + ) - metadata, failed = coordinate_download(self.test_polygon, self.level, self.token) + metadata, failed = coordinate_download( + self.test_polygon, self.level, self.token + ) self.assertTrue(metadata.empty) self.assertFalse(failed.empty) def test_filter_within_time_range(self): - start_time = '2016-01-20 00:00:00' - end_time = '2022-01-21 23:59:59' + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) self.assertEqual(len(filtered_df), 3) - self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) - self.assertTrue(all(filtered_df['captured_at'] <= pd.to_datetime(end_time))) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] <= pd.to_datetime(end_time))) def test_filter_without_end_time(self): - start_time = '2020-01-20 00:00:00' + start_time = "2020-01-20 00:00:00" filtered_df = filter_by_timerange(self.fixture_df, start_time) self.assertEqual(len(filtered_df), 3) - self.assertTrue(all(filtered_df['captured_at'] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) def test_filter_time_no_data(self): - start_time = '2016-01-30 00:00:00' - end_time = '2016-01-31 00:00:00' + start_time = "2016-01-30 00:00:00" + end_time = "2016-01-31 00:00:00" filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) self.assertTrue(filtered_df.empty) @@ -221,10 +264,13 @@ def test_filter_organization_id(self): self.assertEqual(len(filtered_df), 1) def test_filter_time_range(self): - start_time = '2016-01-20 00:00:00' - end_time = '2022-01-21 23:59:59' - filtered_df = filter_results(self.fixture_df, start_time=start_time, end_time=end_time) + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_results( + self.fixture_df, start_time=start_time, end_time=end_time + ) self.assertEqual(len(filtered_df), 3) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 70c97b13d..e70e10db2 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -26,11 +26,11 @@ def test_create_group(self): self.assertTrue(self.project.groups) def test_create_tasks(self): - imageId = self.project.imageList[-1] + imageId = self.project.imageIds[-1] self.project.create_groups() self.project.create_tasks() self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) - #self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) + # self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) if __name__ == "__main__": From c1f1a79a95da699abf496945d4f5749b2064efca Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:20:44 +0100 Subject: [PATCH 017/107] fix: add missing parameter --- .../mapswipe_workers/project_types/street/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 3dd3451d5..9c4845d6a 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -54,7 +54,7 @@ def __init__(self, project_draft): def save_tasks_to_firebase(self, projectId: str, tasks: dict): firebase = Firebase() - firebase.save_tasks_to_firebase(projectId, tasks) + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False) @staticmethod def results_to_postgres(results: dict, project_id: str, filter_mode: bool): From a12a8033e28172af0b9d48c06a69caae81a5ff78 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:37:35 +0100 Subject: [PATCH 018/107] tests(street): add integration test for street project --- .../integration/test_create_street_project.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 mapswipe_workers/tests/integration/test_create_street_project.py diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py new file mode 100644 index 000000000..ecf97c626 --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -0,0 +1,60 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateFootprintProject(unittest.TestCase): + def setUp(self): + self.project_id = [ + set_up.create_test_project_draft("street", "street"), + ] + create_directories() + + def tearDown(self): + for element in self.project_id: + tear_down.delete_test_data(element) + + def test_create_street_project(self): + runner = CliRunner() + result = runner.invoke( + mapswipe_workers.run_create_projects, catch_exceptions=False + ) + if result.exit_code != 0: + raise result.exception + pg_db = auth.postgresDB() + for element in self.project_id: + logger.info(f"Checking project {self.project_id}") + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [element])[0][0] + self.assertEqual(result, element) + + # check if usernames made it to postgres + query = """ + SELECT count(*) + FROM tasks + WHERE project_id = %s + """ + result = pg_db.retr_query(query, [element])[0][0] + self.assertGreater(result, 0) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + # Footprint projects have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + +if __name__ == "__main__": + unittest.main() From 2fb3bc74f866ac9b001edeb1357ce8e6da78f6b4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:38:07 +0100 Subject: [PATCH 019/107] tests(street): add street.json for testing of street project --- .../fixtures/street/projectDrafts/street.json | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json new file mode 100644 index 000000000..2e510f348 --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -0,0 +1,54 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "isPano": false, + "startTimestamp": "2019-07-01T00:00:00.000Z", + "endTimestamp": null, + "organisationId": "1", + "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] +} From 14ff2376fa134f26112c8a4f2d87d9a9d475155d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 10:50:02 +0100 Subject: [PATCH 020/107] fix: test after refactoring --- .../tests/unittests/test_process_mapillary.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 18d9e3401..639295a64 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -50,7 +50,6 @@ def setUpClass(cls): cls.fixture_df = pd.read_csv(file) def setUp(self): - self.token = "test_token" self.level = 14 self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) self.test_multipolygon = MultiPolygon( @@ -174,9 +173,8 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): } row = {"x": 1, "y": 1, "z": 14} - token = "test_token" - result, failed = download_and_process_tile(row, token) + result, failed = download_and_process_tile(row) # Assertions self.assertIsNone(failed) @@ -192,7 +190,7 @@ def test_download_and_process_tile_failure(self, mock_get): mock_get.return_value = mock_response row = pd.Series({"x": 1, "y": 1, "z": self.level}) - result, failed = download_and_process_tile(row, self.token) + result, failed = download_and_process_tile(row) self.assertIsNone(result) self.assertIsNotNone(failed) @@ -205,7 +203,7 @@ def test_coordinate_download(self, mock_download_and_process_tile): ) metadata, failed = coordinate_download( - self.test_polygon, self.level, self.token + self.test_polygon, self.level ) self.assertIsInstance(metadata, pd.DataFrame) @@ -219,7 +217,7 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) ) metadata, failed = coordinate_download( - self.test_polygon, self.level, self.token + self.test_polygon, self.level ) self.assertTrue(metadata.empty) From 80ecf282f8176cb728fd6de9cccca13f9064c3ce Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 11 Nov 2024 15:49:59 +0100 Subject: [PATCH 021/107] feat: add functions for spatial sampling and tests for it --- .../project_types/street/project.py | 1 + .../utils/spatial_sampling.py | 117 ++++++++++++++++++ .../tests/fixtures/mapillary_sequence.csv | 71 +++++++++++ .../tests/unittests/test_spatial_sampling.py | 93 ++++++++++++++ 4 files changed, 282 insertions(+) create mode 100644 mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py create mode 100644 mapswipe_workers/tests/fixtures/mapillary_sequence.csv create mode 100644 mapswipe_workers/tests/unittests/test_spatial_sampling.py diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 9c4845d6a..aa5f1ea48 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -114,6 +114,7 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, + # TODO: change when db allows point geometries data=str(self.imageGeometries.pop()), geometry="", taskId=self.imageIds.pop(), diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py new file mode 100644 index 000000000..8b057e3e6 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -0,0 +1,117 @@ +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +def distance_on_sphere(p1, p2): + """ + p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat + coordinates of the points in set1 and set2 + + Calculate the distance between two points on the Earth's surface using the haversine formula. + + Args: + p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree + + Returns: + numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers. + + This function computes the distance between two points on the Earth's surface using the haversine formula, + which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain + longitude and latitude coordinates in degrees. The function returns an array containing the distances + between corresponding pairs of points. + """ + earth_radius = 6371 # km + + p1 = np.radians(np.array(p1)) + p2 = np.radians(np.array(p2)) + + delta_lat = p2[1] - p1[1] + delta_long = p2[0] - p1[0] + + a = np.sin(delta_lat / 2) ** 2 + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + c = 2 * np.arcsin(np.sqrt(a)) + + distances = earth_radius * c + return distances + +"""-----------------------------------Filtering Points------------------------------------------------""" +def filter_points(df, threshold_distance): + """ + Filter points from a DataFrame based on a threshold distance. + + Args: + df (pandas.DataFrame): DataFrame containing latitude and longitude columns. + threshold_distance (float): Threshold distance for filtering points in kms. + + Returns: + pandas.DataFrame: Filtered DataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function filters points from a DataFrame based on the given threshold distance. It calculates + distances between consecutive points and accumulates them until the accumulated distance surpasses + the threshold distance. It then selects those points and constructs a new DataFrame. Additionally, + it manually checks the last point to include it if it satisfies the length condition. The function + returns the filtered DataFrame along with the calculated road length. + """ + road_length = 0 + mask = np.zeros(len(df), dtype=bool) + mask[0] = True + lat = np.array([wkt.loads(point).y for point in df['data']]) + long = np.array([wkt.loads(point).x for point in df['data']]) + + df['lat'] = lat + df['long'] = long + + + distances = distance_on_sphere([long[1:],lat[1:]], + [long[:-1],lat[:-1]]) + road_length = np.sum(distances) + + #save the last point if the road segment is relavitely small (< 2*road_length) + if threshold_distance <= road_length < 2 * threshold_distance: + mask[-1] = True + + accumulated_distance = 0 + for i, distance in enumerate(distances): + accumulated_distance += distance + if accumulated_distance >= threshold_distance: + mask[i+1] = True + accumulated_distance = 0 # Reset accumulated distance + + to_be_returned_df = df[mask] + # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually + p2 = to_be_returned_df.iloc[0] + distance = distance_on_sphere([float(p2["long"]),float(p2["lat"])],[long[-1],lat[-1]]) + + #last point will be added if it suffices the length condition + #last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) ==1: + to_be_returned_df = pd.concat([to_be_returned_df,pd.DataFrame(df.iloc[-1],columns=to_be_returned_df.columns)],axis=0) + return to_be_returned_df + + +def calculate_spacing(df, interval_length): + """ + Calculate spacing between points in a GeoDataFrame. + + Args: + df (pandas.DataFrame): DataFrame containing points with timestamps. + interval_length (float): Interval length for filtering points in kms. + + Returns: + geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function calculates the spacing between points in a GeoDataFrame by filtering points + based on the provided interval length. It first sorts the GeoDataFrame by timestamp and + then filters points using the filter_points function. The function returns the filtered + GeoDataFrame along with the total road length. + """ + road_length = 0 + if len(df) == 1: + return df + sorted_sub_df = df.sort_values(by=['timestamp']) + filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length) + return filtered_sorted_sub_df diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv new file mode 100644 index 000000000..0d8495a18 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -0,0 +1,71 @@ +,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, +12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, +14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, +18,POINT (38.99769365787506 -6.786351652857817),1453463332000.0,102506575322825.0,1014398349364928,,True,288.49790876724,ywMkSP_5PaJzcbIDa5v1aQ, +19,POINT (38.99540305137634 -6.785360860858361),1453463350000.0,102506575322825.0,165035685525369,,True,293.57188365181,ywMkSP_5PaJzcbIDa5v1aQ, +20,POINT (39.00015592575073 -6.787491593818643),1453463300000.0,102506575322825.0,1139870886511452,,True,305.58569254109,ywMkSP_5PaJzcbIDa5v1aQ, +21,POINT (38.9979350566864 -6.7864262285172),1453463330000.0,102506575322825.0,921248855101166,,True,287.76018227153,ywMkSP_5PaJzcbIDa5v1aQ, +22,POINT (38.99890601634979 -6.786804433469129),1453463318000.0,102506575322825.0,233272625257058,,True,295.57372041131,ywMkSP_5PaJzcbIDa5v1aQ, +23,POINT (38.99618625640869 -6.7857071056060505),1453463344000.0,102506575322825.0,762184237820558,,True,293.88722229187,ywMkSP_5PaJzcbIDa5v1aQ, +26,POINT (38.98969531059265 -6.782905179427416),1453463424000.0,102506575322825.0,1099193250486391,,True,292.87773588969,ywMkSP_5PaJzcbIDa5v1aQ, +27,POINT (38.998627066612244 -6.786671262745287),1453463322000.0,102506575322825.0,831521697449190,,True,296.54786608582,ywMkSP_5PaJzcbIDa5v1aQ, +29,POINT (38.992629647254944 -6.784167646282242),1453463368000.0,102506575322825.0,151617226911336,,True,292.93390738544,ywMkSP_5PaJzcbIDa5v1aQ, +30,POINT (39.00003254413605 -6.7873584232849),1453463302000.0,102506575322825.0,1773698542801178,,True,296.75481871036,ywMkSP_5PaJzcbIDa5v1aQ, +32,POINT (39.000563621520996 -6.787811202949342),1453463298000.0,102506575322825.0,164116422292333,,True,305.80313236719,ywMkSP_5PaJzcbIDa5v1aQ, +37,POINT (38.9902800321579 -6.783144888578377),1453463410000.0,102506575322825.0,329957465177767,,True,293.87481737883,ywMkSP_5PaJzcbIDa5v1aQ, +38,POINT (38.99117052555084 -6.783539076700606),1453463388000.0,102506575322825.0,2948533428757015,,True,294.44274652914,ywMkSP_5PaJzcbIDa5v1aQ, +39,POINT (38.99877190589905 -6.78674051152629),1453463320000.0,102506575322825.0,479325670049740,,True,296.24340777377,ywMkSP_5PaJzcbIDa5v1aQ, +41,POINT (38.990601897239685 -6.783294040878786),1453463402000.0,102506575322825.0,501560441022559,,True,289.98678899102,ywMkSP_5PaJzcbIDa5v1aQ, +42,POINT (38.989362716674805 -6.7828785450699485),1453463432000.0,102506575322825.0,494436578424418,,True,249.25945175736,ywMkSP_5PaJzcbIDa5v1aQ, +44,POINT (38.994566202163696 -6.7850039621655895),1453463356000.0,102506575322825.0,2928848347373461,,True,295.93075138027,ywMkSP_5PaJzcbIDa5v1aQ, +45,POINT (38.993815183639526 -6.784657716912335),1453463362000.0,102506575322825.0,167884815220625,,True,290.65289338004,ywMkSP_5PaJzcbIDa5v1aQ, +47,POINT (38.991841077804565 -6.783837381011111),1453463380000.0,102506575322825.0,2783373888590755,,True,292.94668882511,ywMkSP_5PaJzcbIDa5v1aQ, +48,POINT (38.99052679538727 -6.783262079675453),1453463404000.0,102506575322825.0,500930794384261,,True,296.09431621523,ywMkSP_5PaJzcbIDa5v1aQ, +49,POINT (38.9897757768631 -6.782937140654425),1453463422000.0,102506575322825.0,473363863989539,,True,292.22088072734,ywMkSP_5PaJzcbIDa5v1aQ, +50,POINT (38.99429798126221 -6.784870790943785),1453463358000.0,102506575322825.0,792308461667709,,True,295.36479453372,ywMkSP_5PaJzcbIDa5v1aQ, +51,POINT (38.9997535943985 -6.787219925890724),1453463306000.0,102506575322825.0,1169832606865116,,True,296.51874301031,ywMkSP_5PaJzcbIDa5v1aQ, +54,POINT (38.992860317230225 -6.784268856561923),1453463364000.0,102506575322825.0,143904254368287,,True,292.87299883627,ywMkSP_5PaJzcbIDa5v1aQ, +57,POINT (38.98994743824005 -6.783006389972371),1453463418000.0,102506575322825.0,512708183243254,,True,292.58758044439,ywMkSP_5PaJzcbIDa5v1aQ, +58,POINT (38.99670124053955 -6.785941486524692),1453463340000.0,102506575322825.0,168474601828325,,True,294.32908734047,ywMkSP_5PaJzcbIDa5v1aQ, +59,POINT (38.992136120796204 -6.783959898799395),1453463376000.0,102506575322825.0,171815874817246,,True,292.47687051975,ywMkSP_5PaJzcbIDa5v1aQ, +60,POINT (38.99090766906738 -6.783405905073778),1453463394000.0,102506575322825.0,475809606904698,,True,297.28158189053,ywMkSP_5PaJzcbIDa5v1aQ, +61,POINT (38.99251699447632 -6.7841250314212544),1453463370000.0,102506575322825.0,798930200741228,,True,292.63573224675,ywMkSP_5PaJzcbIDa5v1aQ, +62,POINT (38.99019956588745 -6.7831129273651385),1453463412000.0,102506575322825.0,989719805170705,,True,292.67075887406,ywMkSP_5PaJzcbIDa5v1aQ, +63,POINT (38.991336822509766 -6.783613652795566),1453463386000.0,102506575322825.0,887351401825944,,True,294.11111203184,ywMkSP_5PaJzcbIDa5v1aQ, +64,POINT (38.99745762348175 -6.786271750352796),1453463334000.0,102506575322825.0,820185135568044,,True,292.38261450394,ywMkSP_5PaJzcbIDa5v1aQ, +69,POINT (38.99919033050537 -6.7869376041561225),1453463314000.0,102506575322825.0,1401323273568889,,True,296.07037190303,ywMkSP_5PaJzcbIDa5v1aQ, +70,POINT (38.99229168891907 -6.784023821111347),1453463374000.0,102506575322825.0,971311816939999,,True,293.93020293096,ywMkSP_5PaJzcbIDa5v1aQ, +71,POINT (38.9956659078598 -6.785483378259087),1453463348000.0,102506575322825.0,1888667171315269,,True,294.14026651816,ywMkSP_5PaJzcbIDa5v1aQ, +77,POINT (38.99095058441162 -6.783437866267576),1453463392000.0,102506575322825.0,313013763568992,,True,299.51413646126,ywMkSP_5PaJzcbIDa5v1aQ, +78,POINT (38.99240434169769 -6.784077089698172),1453463372000.0,102506575322825.0,300316491494684,,True,293.91812405268,ywMkSP_5PaJzcbIDa5v1aQ, +79,POINT (38.994094133377075 -6.784774907641307),1453463360000.0,102506575322825.0,799986720927690,,True,293.50534124709,ywMkSP_5PaJzcbIDa5v1aQ, +80,POINT (38.998122811317444 -6.786495477333432),1453463328000.0,102506575322825.0,766866250678914,,True,291.66896324881,ywMkSP_5PaJzcbIDa5v1aQ, +82,POINT (38.99715185165405 -6.786143906317193),1453463336000.0,102506575322825.0,4104687166260249,,True,294.20932475917,ywMkSP_5PaJzcbIDa5v1aQ, +84,POINT (38.99081647396088 -6.783368617011661),1453463396000.0,102506575322825.0,311476673697450,,True,292.1387426333,ywMkSP_5PaJzcbIDa5v1aQ, +85,POINT (38.999614119529724 -6.787150677178687),1453463308000.0,102506575322825.0,486545769438431,,True,296.75245798459,ywMkSP_5PaJzcbIDa5v1aQ, +86,POINT (38.99830520153046 -6.786516784659511),1453463326000.0,102506575322825.0,287818266220456,,True,296.90157475174,ywMkSP_5PaJzcbIDa5v1aQ, +89,POINT (38.991073966026306 -6.78349646178404),1453463390000.0,102506575322825.0,1104648166685558,,True,294.64381621501,ywMkSP_5PaJzcbIDa5v1aQ, +95,POINT (38.99045169353485 -6.783224791602194),1453463406000.0,102506575322825.0,1189096484862313,,True,296.52075233925,ywMkSP_5PaJzcbIDa5v1aQ, +96,POINT (38.989604115486145 -6.782867891326546),1453463426000.0,102506575322825.0,149492403718498,,True,291.35167681135,ywMkSP_5PaJzcbIDa5v1aQ, +99,POINT (38.990371227264404 -6.783187503526065),1453463408000.0,102506575322825.0,1891398304374063,,True,293.92928705821,ywMkSP_5PaJzcbIDa5v1aQ, +102,POINT (38.99695873260498 -6.786058676941238),1453463338000.0,102506575322825.0,823059255257426,,True,294.12835748694,ywMkSP_5PaJzcbIDa5v1aQ, +105,POINT (38.99198055267334 -6.7838959764789735),1453463378000.0,102506575322825.0,3696263247264941,,True,293.15037639266,ywMkSP_5PaJzcbIDa5v1aQ, +108,POINT (38.989861607551575 -6.782974428749938),1453463420000.0,102506575322825.0,862036401042035,,True,291.44770311405,ywMkSP_5PaJzcbIDa5v1aQ, +109,POINT (38.98951292037964 -6.782841256967004),1453463428000.0,102506575322825.0,304865891166877,,True,272.6464361279,ywMkSP_5PaJzcbIDa5v1aQ, +111,POINT (38.99274230003357 -6.784220914853137),1453463366000.0,102506575322825.0,794241458144562,,True,293.2506353664,ywMkSP_5PaJzcbIDa5v1aQ, +113,POINT (38.99847149848938 -6.786596687123861),1453463324000.0,102506575322825.0,939866586773160,,True,295.61868532377,ywMkSP_5PaJzcbIDa5v1aQ, +115,POINT (38.99947464466095 -6.787081428456702),1453463310000.0,102506575322825.0,1143620816058871,,True,296.46351643457,ywMkSP_5PaJzcbIDa5v1aQ, +116,POINT (38.99003326892853 -6.783043678062526),1453463416000.0,102506575322825.0,1405161323178994,,True,293.46012249789,ywMkSP_5PaJzcbIDa5v1aQ, +119,POINT (39.00104105472565 -6.78785914430064),1453463296000.0,102506575322825.0,1273957886333326,,True,279.89824953791,ywMkSP_5PaJzcbIDa5v1aQ, +120,POINT (38.98943245410919 -6.782857237582903),1453463430000.0,102506575322825.0,1171907646606759,,True,252.08147683237,ywMkSP_5PaJzcbIDa5v1aQ, +121,POINT (38.99148166179657 -6.783677575153462),1453463384000.0,102506575322825.0,164230852280713,,True,293.89449177018,ywMkSP_5PaJzcbIDa5v1aQ, +122,POINT (38.999335169792175 -6.787012179724755),1453463312000.0,102506575322825.0,510570906624460,,True,297.86598575521,ywMkSP_5PaJzcbIDa5v1aQ, +124,POINT (38.99592339992523 -6.785595241945558),1453463346000.0,102506575322825.0,136548521821574,,True,293.97664288588,ywMkSP_5PaJzcbIDa5v1aQ, +125,POINT (38.990119099617004 -6.783075639280355),1453463414000.0,102506575322825.0,1112069379291585,,True,293.19285658924,ywMkSP_5PaJzcbIDa5v1aQ, +126,POINT (38.99644374847412 -6.785829622918669),1453463342000.0,102506575322825.0,846432512639247,,True,294.78265773594,ywMkSP_5PaJzcbIDa5v1aQ, +128,POINT (38.99165332317352 -6.783752151226963),1453463382000.0,102506575322825.0,133891088754131,,True,294.07658010782,ywMkSP_5PaJzcbIDa5v1aQ, +129,POINT (38.99078965187073 -6.7833579632791015),1453463398000.0,102506575322825.0,211793933759059,,True,294.17362400652,ywMkSP_5PaJzcbIDa5v1aQ, +131,POINT (38.99904549121857 -6.786873682230961),1453463316000.0,102506575322825.0,137437058395340,,True,295.32783278431,ywMkSP_5PaJzcbIDa5v1aQ, +132,POINT (38.99989306926727 -6.787289174592786),1453463304000.0,102506575322825.0,323401522460093,,True,296.4023335068,ywMkSP_5PaJzcbIDa5v1aQ, +133,POINT (38.994882702827454 -6.785147787043741),1453463354000.0,102506575322825.0,1070660863461711,,True,294.24600460049,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py new file mode 100644 index 000000000..38322dfc8 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -0,0 +1,93 @@ +import os + +import unittest +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, calculate_spacing + + +class TestDistanceCalculations(unittest.TestCase): + + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_sequence.csv", + ), + "r", + ) as file: + cls.fixture_df = pd.read_csv(file) + + def test_distance_on_sphere(self): + p1 = Point(-74.006, 40.7128) + p2 = Point(-118.2437, 34.0522) + + distance = distance_on_sphere((p1.x, p1.y), (p2.x, p2.y)) + expected_distance = 3940 # Approximate known distance in km + + self.assertTrue(np.isclose(distance, expected_distance, atol=50)) + + def test_filter_points(self): + data = { + "data": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ] + } + df = pd.DataFrame(data) + + threshold_distance = 100 + filtered_df = filter_points(df, threshold_distance) + + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLessEqual(len(filtered_df), len(df)) + + + def test_calculate_spacing(self): + data = { + "data": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)" + ], + 'timestamp': [1, 2, 3, 4] + } + df = pd.DataFrame(data) + gdf = pd.DataFrame(df) + + interval_length = 100 + filtered_gdf = calculate_spacing(gdf, interval_length) + + self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing) + + + def test_calculate_spacing_with_sequence(self): + threshold_distance = 5 + filtered_df = filter_points(self.fixture_df, threshold_distance) + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLess(len(filtered_df), len(self.fixture_df)) + + filtered_df.reset_index(drop=True, inplace=True) + + for i in range(len(filtered_df) - 1): + geom1 = wkt.loads(filtered_df.loc[i, 'data']) + geom2 = wkt.loads(filtered_df.loc[i + 1, 'data']) + + distance = geom1.distance(geom2) + + self.assertLess(distance, threshold_distance) + + + + +if __name__ == "__main__": + unittest.main() From 984d8213927d437655fe0821222e9ee4738f3a0a Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 14 Nov 2024 17:00:55 +0100 Subject: [PATCH 022/107] feat: use filtering in street project --- .../project_types/street/project.py | 10 ++- .../utils/process_mapillary.py | 44 +++++++++--- .../tests/fixtures/projectDrafts/street.json | 4 +- .../tests/unittests/test_process_mapillary.py | 68 +++++++++++++++++-- .../unittests/test_project_type_street.py | 18 ++++- 5 files changed, 124 insertions(+), 20 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index aa5f1ea48..cb2e83763 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -46,7 +46,15 @@ def __init__(self, project_draft): self.tasks: Dict[str, List[StreetTask]] = {} self.geometry = project_draft["geometry"] - ImageMetadata = get_image_metadata(self.geometry) + + # TODO: validate inputs + ImageMetadata = get_image_metadata( + self.geometry, + is_pano=project_draft.get("isPano", None), + start_time=project_draft.get("startTimestamp", None), + end_time=project_draft.get("endTimestamp", None), + organization_id=project_draft.get("organizationId", None), + ) self.imageIds = ImageMetadata["ids"] diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 4f1a55084..5a2ba53c2 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -17,6 +17,7 @@ from vt2geojson import tools as vt2geojson_tools from concurrent.futures import ThreadPoolExecutor, as_completed from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY +from mapswipe_workers.definitions import logger def create_tiles(polygon, level): @@ -128,6 +129,12 @@ def coordinate_download( failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( drop=True ) + target_columns = [ + "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" + ] + for col in target_columns: + if col not in downloaded_metadata.columns: + downloaded_metadata[col] = None return downloaded_metadata, failed_tiles @@ -157,11 +164,11 @@ def geojson_to_polygon(geojson_data): def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") - start_time = pd.Timestamp(start_time) - + start_time = pd.to_datetime(start_time).tz_localize(None) if end_time is None: - end_time = pd.Timestamp.now() - + end_time = pd.Timestamp.now().tz_localize(None) + else: + end_time = pd.to_datetime(end_time).tz_localize(None) filtered_df = df[ (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) ] @@ -177,12 +184,30 @@ def filter_results( ): df = results_df.copy() if is_pano is not None: + if df["is_pano"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'is_pano' value." + ) + return None df = df[df["is_pano"] == is_pano] + if organization_id is not None: + if df["organization_id"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has an 'organization_id' value." + ) + return None df = df[df["organization_id"] == organization_id] + if start_time is not None: + if df["captured_at"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'captured_at' value." + ) + return None df = filter_by_timerange(df, start_time, end_time) + return df @@ -202,7 +227,10 @@ def get_image_metadata( downloaded_metadata = filter_results( downloaded_metadata, is_pano, organization_id, start_time, end_time ) - return { - "ids": downloaded_metadata["id"].tolist(), - "geometries": downloaded_metadata["geometry"].tolist(), - } + if downloaded_metadata.isna().all().all() == False: + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } + else: + raise ValueError("No Mapillary Features in the AoI match the filter criteria.") diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 341756584..3093e489c 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -47,7 +47,5 @@ "verificationNumber": 3, "groupSize": 25, "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z", - "endTimestamp": null, - "organisationId": "1" + "startTimestamp": "2019-07-01T00:00:00.000Z" } diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 639295a64..caa9e76d6 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -18,12 +18,10 @@ geojson_to_polygon, filter_by_timerange, filter_results, + get_image_metadata, ) -# Assuming create_tiles, download_and_process_tile, and coordinate_download are imported - - class TestTileGroupingFunctions(unittest.TestCase): @classmethod def setUpClass(cls): @@ -156,13 +154,11 @@ def test_geojson_to_polygon_contribution_geojson(self): ) @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): - # Mock the response from requests.get mock_response = MagicMock() mock_response.status_code = 200 mock_response.content = b"mock vector tile data" # Example mock data mock_get.return_value = mock_response - # Mock the return value of vt_bytes_to_geojson mock_vt2geojson.return_value = { "features": [ { @@ -176,7 +172,6 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): result, failed = download_and_process_tile(row) - # Assertions self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) @@ -269,6 +264,67 @@ def test_filter_time_range(self): ) self.assertEqual(len(filtered_df), 3) + def test_filter_no_rows_after_filter(self): + filtered_df = filter_results(self.fixture_df, is_pano="False") + self.assertTrue(filtered_df.empty) + + def test_filter_missing_columns(self): + columns_to_check = ["is_pano", "organization_id", "captured_at"] # Add your column names here + for column in columns_to_check: + df_copy = self.fixture_df.copy() + df_copy[column] = None + if column == "captured_at": + column = "start_time" + + result = filter_results(df_copy, **{column: True}) + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + result = get_image_metadata(self.fixture_data) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_filtering(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + params = { + "is_pano": True, + "start_time": "2016-01-20 00:00:00", + "end_time": "2022-01-21 23:59:59", + } + + result = get_image_metadata(self.fixture_data, **params) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_no_rows(self, mock_coordinate_download): + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + params = { + "is_pano": True, + "start_time": "1916-01-20 00:00:00", + "end_time": "1922-01-21 23:59:59", + } + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data, **params) + if __name__ == "__main__": unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index e70e10db2..7cc56057c 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -2,6 +2,7 @@ import os import unittest from unittest.mock import patch +import pandas as pd from mapswipe_workers.project_types import StreetProject from tests import fixtures @@ -16,7 +17,21 @@ def setUp(self) -> None: ) ) project_draft["projectDraftId"] = "foo" - self.project = StreetProject(project_draft) + + with patch( + "mapswipe_workers.utils.process_mapillary.coordinate_download" + ) as mock_get: + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + mock_get.return_value = (pd.read_csv(file), None) + self.project = StreetProject(project_draft) def test_init(self): self.assertEqual(self.project.geometry["type"], "FeatureCollection") @@ -30,7 +45,6 @@ def test_create_tasks(self): self.project.create_groups() self.project.create_tasks() self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) - # self.assertEqual(self.project.groups["g0"].numberOfTasks, 1) if __name__ == "__main__": From 4ab528dcd463f666e54669b58fb070f025b912ef Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 15:29:11 +0100 Subject: [PATCH 023/107] feat: add spatial sampling to street project --- .../project_types/street/project.py | 1 + .../utils/process_mapillary.py | 8 ++++ .../utils/spatial_sampling.py | 30 +++++++++----- .../tests/fixtures/mapillary_sequence.csv | 2 +- .../tests/fixtures/projectDrafts/street.json | 4 +- .../fixtures/street/projectDrafts/street.json | 4 -- .../tests/unittests/test_process_mapillary.py | 5 ++- .../unittests/test_project_type_street.py | 6 ++- .../tests/unittests/test_spatial_sampling.py | 39 +++++++++++-------- 9 files changed, 64 insertions(+), 35 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index cb2e83763..f7184897c 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -54,6 +54,7 @@ def __init__(self, project_draft): start_time=project_draft.get("startTimestamp", None), end_time=project_draft.get("endTimestamp", None), organization_id=project_draft.get("organizationId", None), + sampling_threshold=project_draft.get("samplingThreshold", None), ) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5a2ba53c2..0e8f8a4a1 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -18,6 +18,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.spatial_sampling import spatial_sampling def create_tiles(polygon, level): @@ -219,14 +220,21 @@ def get_image_metadata( organization_id: str = None, start_time: str = None, end_time: str = None, + sampling_threshold = None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata, failed_tiles = coordinate_download( aoi_polygon, level, attempt_limit ) + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda geom: isinstance(geom, Point)) + ] + downloaded_metadata = filter_results( downloaded_metadata, is_pano, organization_id, start_time, end_time ) + if sampling_threshold is not None: + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if downloaded_metadata.isna().all().all() == False: return { "ids": downloaded_metadata["id"].tolist(), diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 8b057e3e6..082346b96 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -58,11 +58,8 @@ def filter_points(df, threshold_distance): road_length = 0 mask = np.zeros(len(df), dtype=bool) mask[0] = True - lat = np.array([wkt.loads(point).y for point in df['data']]) - long = np.array([wkt.loads(point).x for point in df['data']]) - - df['lat'] = lat - df['long'] = long + lat = df["lat"].to_numpy() + long = df["long"].to_numpy() distances = distance_on_sphere([long[1:],lat[1:]], @@ -92,7 +89,7 @@ def filter_points(df, threshold_distance): return to_be_returned_df -def calculate_spacing(df, interval_length): +def spatial_sampling(df, interval_length): """ Calculate spacing between points in a GeoDataFrame. @@ -109,9 +106,22 @@ def calculate_spacing(df, interval_length): then filters points using the filter_points function. The function returns the filtered GeoDataFrame along with the total road length. """ - road_length = 0 if len(df) == 1: return df - sorted_sub_df = df.sort_values(by=['timestamp']) - filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length) - return filtered_sorted_sub_df + + df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) + df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) + sorted_df = df.sort_values(by=['captured_at']) + + sampled_sequence_df = pd.DataFrame() + + # loop through each sequence + for sequence in sorted_df['sequence_id'].unique(): + sequence_df = sorted_df[sorted_df['sequence_id'] == sequence] + + filtered_sorted_sub_df = filter_points(sequence_df,interval_length) + sampled_sequence_df = pd.concat([sampled_sequence_df,filtered_sorted_sub_df],axis=0) + + + + return sampled_sequence_df diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv index 0d8495a18..597fa6f66 100644 --- a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -1,4 +1,4 @@ -,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +,geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id 9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, 12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, 14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 3093e489c..1dd5b452a 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -46,6 +46,6 @@ "requestingOrganisation": "test", "verificationNumber": 3, "groupSize": 25, - "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z" + "startTimestamp": "2019-07-01T00:00:00.000Z", + "samplingThreshold": 0.1 } diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json index 2e510f348..f945f1661 100644 --- a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -46,9 +46,5 @@ "requestingOrganisation": "test", "verificationNumber": 3, "groupSize": 25, - "isPano": false, - "startTimestamp": "2019-07-01T00:00:00.000Z", - "endTimestamp": null, - "organisationId": "1", "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] } diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index caa9e76d6..5bec093d3 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -9,6 +9,7 @@ MultiLineString, GeometryCollection, ) +from shapely import wkt import pandas as pd from unittest.mock import patch, MagicMock from mapswipe_workers.utils.process_mapillary import ( @@ -45,7 +46,9 @@ def setUpClass(cls): ), "r", ) as file: - cls.fixture_df = pd.read_csv(file) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + cls.fixture_df = df def setUp(self): self.level = 14 diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 7cc56057c..64b80b3e7 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -2,6 +2,7 @@ import os import unittest from unittest.mock import patch +from shapely import wkt import pandas as pd from mapswipe_workers.project_types import StreetProject @@ -30,7 +31,10 @@ def setUp(self) -> None: ), "r", ) as file: - mock_get.return_value = (pd.read_csv(file), None) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + mock_get.return_value = (df, None) self.project = StreetProject(project_draft) def test_init(self): diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py index 38322dfc8..f43597c89 100644 --- a/mapswipe_workers/tests/unittests/test_spatial_sampling.py +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -6,7 +6,7 @@ from shapely import wkt from shapely.geometry import Point -from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, calculate_spacing +from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, spatial_sampling class TestDistanceCalculations(unittest.TestCase): @@ -22,7 +22,10 @@ def setUpClass(cls): ), "r", ) as file: - cls.fixture_df = pd.read_csv(file) + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + cls.fixture_df = df def test_distance_on_sphere(self): p1 = Point(-74.006, 40.7128) @@ -35,7 +38,7 @@ def test_distance_on_sphere(self): def test_filter_points(self): data = { - "data": [ + "geometry": [ "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", @@ -44,6 +47,10 @@ def test_filter_points(self): } df = pd.DataFrame(data) + df['geometry'] = df['geometry'].apply(wkt.loads) + + df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) + df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) threshold_distance = 100 filtered_df = filter_points(df, threshold_distance) @@ -51,36 +58,36 @@ def test_filter_points(self): self.assertLessEqual(len(filtered_df), len(df)) - def test_calculate_spacing(self): + def test_spatial_sampling_ordering(self): data = { - "data": [ + "geometry": [ "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", "POINT (-77.006 43.7128)" ], - 'timestamp': [1, 2, 3, 4] + 'captured_at': [1, 2, 3, 4], + 'sequence_id': ['1', '1', '1', '1'] } df = pd.DataFrame(data) - gdf = pd.DataFrame(df) + df['geometry'] = df['geometry'].apply(wkt.loads) - interval_length = 100 - filtered_gdf = calculate_spacing(gdf, interval_length) + interval_length = 0.1 + filtered_gdf = spatial_sampling(df, interval_length) - self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing) + self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing) - def test_calculate_spacing_with_sequence(self): - threshold_distance = 5 - filtered_df = filter_points(self.fixture_df, threshold_distance) + def test_spatial_sampling_with_sequence(self): + threshold_distance = 0.01 + filtered_df = spatial_sampling(self.fixture_df, threshold_distance) self.assertIsInstance(filtered_df, pd.DataFrame) self.assertLess(len(filtered_df), len(self.fixture_df)) filtered_df.reset_index(drop=True, inplace=True) - for i in range(len(filtered_df) - 1): - geom1 = wkt.loads(filtered_df.loc[i, 'data']) - geom2 = wkt.loads(filtered_df.loc[i + 1, 'data']) + geom1 = filtered_df.loc[i, 'geometry'] + geom2 = filtered_df.loc[i + 1, 'geometry'] distance = geom1.distance(geom2) From f7fc5f3123f1690e562e535a7a92324c54eea55e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:42:32 +0100 Subject: [PATCH 024/107] feat: add size restriction for too many images --- .../utils/process_mapillary.py | 16 ++++++++----- .../tests/unittests/test_process_mapillary.py | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 0e8f8a4a1..a762190d6 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -208,7 +208,6 @@ def filter_results( return None df = filter_by_timerange(df, start_time, end_time) - return df @@ -235,10 +234,15 @@ def get_image_metadata( ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - if downloaded_metadata.isna().all().all() == False: - return { - "ids": downloaded_metadata["id"].tolist(), - "geometries": downloaded_metadata["geometry"].tolist(), - } + if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == False: + if len(downloaded_metadata) > 100000: + err = (f"Too many Images with selected filter " + f"options for the AoI: {len(downloaded_metadata)}") + raise ValueError(err) + else: + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } else: raise ValueError("No Mapillary Features in the AoI match the filter criteria.") diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 5bec093d3..f6a7e3fdb 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -328,6 +328,30 @@ def test_get_image_metadata_no_rows(self, mock_coordinate_download): with self.assertRaises(ValueError): get_image_metadata(self.fixture_data, **params) + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_empty_response(self, mock_coordinate_download): + df = self.fixture_df.copy() + df = df.drop(df.index) + mock_coordinate_download.return_value = ( + df, + None + ) + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + + @patch("mapswipe_workers.utils.process_mapillary.filter_results") + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): + mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) + mock_coordinate_download.return_value = ( + self.fixture_df, + None, + ) + + with self.assertRaises(ValueError): + get_image_metadata(self.fixture_data) + if __name__ == "__main__": unittest.main() From f11f75edf271b08df2c564d7d42f58be0e711a3f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:44:23 +0100 Subject: [PATCH 025/107] breaking: change geom column in postgres to allow all geometry types --- .../v2_to_v3/08_change_geom_type_for_tasks.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql diff --git a/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql new file mode 100644 index 000000000..97ecc298f --- /dev/null +++ b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql @@ -0,0 +1,12 @@ +/* + * This script updates the `tasks` table by adjusting the following column: + * - `geom`: Stores now not only polygon geometries but all geometry types (e.g. Point, LineString). + * + * Existing entries for `geom` are not affected by this change. + * + * The new street project type requires Point geometries to store the image location. + * + */ + + +ALTER TABLE tasks ALTER COLUMN geom SET DATA TYPE geometry(Geometry, 4326); \ No newline at end of file From 98f82c936e0143da89fe0b4bd0149eb63db06453 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 17:46:35 +0100 Subject: [PATCH 026/107] feat: save location of image as geometry --- django/apps/existing_database/models.py | 2 +- .../mapswipe_workers/project_types/street/project.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 0cf6582fb..ef65496b0 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -127,7 +127,7 @@ class Task(Model): project = models.ForeignKey(Project, models.DO_NOTHING, related_name="+") group_id = models.CharField(max_length=999) task_id = models.CharField(max_length=999) - geom = gis_models.MultiPolygonField(blank=True, null=True) + geom = gis_models.GeometryField(blank=True, null=True) # Database uses JSON instead of JSONB (not supported by django) project_type_specifics = models.TextField(blank=True, null=True) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index f7184897c..1853755df 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -36,7 +36,6 @@ class StreetGroup(BaseGroup): @dataclass class StreetTask(BaseTask): geometry: str - data: str class StreetProject(BaseProject): @@ -123,9 +122,7 @@ def create_tasks(self): task = StreetTask( projectId=self.projectId, groupId=group_id, - # TODO: change when db allows point geometries - data=str(self.imageGeometries.pop()), - geometry="", + geometry=self.imageGeometries.pop(), taskId=self.imageIds.pop(), ) self.tasks[group_id].append(task) From 6a1d15e789f6dc7785a09dc7a9e3a1e15f42bd94 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 18 Nov 2024 18:11:47 +0100 Subject: [PATCH 027/107] feat: use spatial filter on downloaded images --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index a762190d6..5773792c4 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -137,6 +137,10 @@ def coordinate_download( if col not in downloaded_metadata.columns: downloaded_metadata[col] = None + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + ] + return downloaded_metadata, failed_tiles From 05f1449954fe85e4a645122f197f49616e2cde7d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 12:29:37 +0100 Subject: [PATCH 028/107] fix: adapt mapillary token entry in example env --- example.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example.env b/example.env index e3b856ea2..2fa92ae33 100644 --- a/example.env +++ b/example.env @@ -77,4 +77,4 @@ COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org # Mapillary -MAPILLARY_ACCESS_TOKEN= \ No newline at end of file +MAPILLARY_API_KEY= \ No newline at end of file From 263d738adeb05e5175760333869e4cbdb707c1cc Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 13:34:33 +0100 Subject: [PATCH 029/107] fix: do not return failed_rows and do not use functions on empty df --- .../utils/process_mapillary.py | 13 ++++---- .../tests/unittests/test_process_mapillary.py | 30 +++++-------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5773792c4..5d83b89f5 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -125,7 +125,7 @@ def coordinate_download( if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: - downloaded_metadata = pd.DataFrame(downloaded_metadata) + return pd.DataFrame(downloaded_metadata) failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( drop=True @@ -137,11 +137,12 @@ def coordinate_download( if col not in downloaded_metadata.columns: downloaded_metadata[col] = None - downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) - ] + if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == True: + downloaded_metadata = downloaded_metadata[ + downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + ] - return downloaded_metadata, failed_tiles + return downloaded_metadata def geojson_to_polygon(geojson_data): @@ -226,7 +227,7 @@ def get_image_metadata( sampling_threshold = None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) - downloaded_metadata, failed_tiles = coordinate_download( + downloaded_metadata = coordinate_download( aoi_polygon, level, attempt_limit ) downloaded_metadata = downloaded_metadata[ diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index f6a7e3fdb..b6429417b 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -200,7 +200,7 @@ def test_coordinate_download(self, mock_download_and_process_tile): None, ) - metadata, failed = coordinate_download( + metadata = coordinate_download( self.test_polygon, self.level ) @@ -214,12 +214,11 @@ def test_coordinate_download_with_failures(self, mock_download_and_process_tile) pd.Series({"x": 1, "y": 1, "z": self.level}), ) - metadata, failed = coordinate_download( + metadata = coordinate_download( self.test_polygon, self.level ) self.assertTrue(metadata.empty) - self.assertFalse(failed.empty) def test_filter_within_time_range(self): start_time = "2016-01-20 00:00:00" @@ -284,10 +283,7 @@ def test_filter_missing_columns(self): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df result = get_image_metadata(self.fixture_data) self.assertIsInstance(result, dict) self.assertIn("ids", result) @@ -296,10 +292,7 @@ def test_get_image_metadata(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_filtering(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df params = { "is_pano": True, @@ -315,10 +308,7 @@ def test_get_image_metadata_filtering(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_no_rows(self, mock_coordinate_download): - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df params = { "is_pano": True, @@ -332,10 +322,7 @@ def test_get_image_metadata_no_rows(self, mock_coordinate_download): def test_get_image_metadata_empty_response(self, mock_coordinate_download): df = self.fixture_df.copy() df = df.drop(df.index) - mock_coordinate_download.return_value = ( - df, - None - ) + mock_coordinate_download.return_value = df with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) @@ -344,10 +331,7 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) - mock_coordinate_download.return_value = ( - self.fixture_df, - None, - ) + mock_coordinate_download.return_value = self.fixture_df with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) From df17d3c9e9b3ecab9f252f02ca11d4a8c133596f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 13:38:17 +0100 Subject: [PATCH 030/107] fix: testing for removed funcionality --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ---- mapswipe_workers/tests/unittests/test_process_mapillary.py | 1 - 2 files changed, 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 5d83b89f5..3558e9b38 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -121,15 +121,11 @@ def coordinate_download( downloaded_metadata.append(df) if failed_row is not None: failed_tiles.append(failed_row) - if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: return pd.DataFrame(downloaded_metadata) - failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index( - drop=True - ) target_columns = [ "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" ] diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index b6429417b..8f918f0c0 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -205,7 +205,6 @@ def test_coordinate_download(self, mock_download_and_process_tile): ) self.assertIsInstance(metadata, pd.DataFrame) - self.assertTrue(failed.empty) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): From 9f2ac813c771f50a9e23bb9488263d388415d4e1 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 14:11:41 +0100 Subject: [PATCH 031/107] style: code formatting --- .../project_types/street/project.py | 11 ++-- .../utils/process_mapillary.py | 40 ++++++++----- .../utils/spatial_sampling.py | 59 ++++++++++++------- 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index 1853755df..fba87f390 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -21,7 +21,7 @@ build_multipolygon_from_layer_geometries, check_if_layer_has_too_many_geometries, save_geojson_to_file, - multipolygon_to_wkt + multipolygon_to_wkt, ) from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup from mapswipe_workers.utils.process_mapillary import get_image_metadata @@ -56,7 +56,6 @@ def __init__(self, project_draft): sampling_threshold=project_draft.get("samplingThreshold", None), ) - self.imageIds = ImageMetadata["ids"] self.imageGeometries = ImageMetadata["geometries"] @@ -83,7 +82,9 @@ def validate_geometries(self): self.inputGeometriesFileName = save_geojson_to_file( self.projectId, self.geometry ) - layer, datasource = load_geojson_to_ogr(self.projectId, self.inputGeometriesFileName) + layer, datasource = load_geojson_to_ogr( + self.projectId, self.inputGeometriesFileName + ) # check if inputs fit constraints check_if_layer_is_empty(self.projectId, layer) @@ -97,7 +98,9 @@ def validate_geometries(self): del datasource del layer - logger.info(f"{self.projectId}" f" - validate geometry - " f"input geometry is correct.") + logger.info( + f"{self.projectId}" f" - validate geometry - " f"input geometry is correct." + ) wkt_geometry = multipolygon_to_wkt(multi_polygon) return wkt_geometry diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 3558e9b38..8181006e7 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -127,15 +127,26 @@ def coordinate_download( return pd.DataFrame(downloaded_metadata) target_columns = [ - "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id" + "id", + "geometry", + "captured_at", + "is_pano", + "compass_angle", + "sequence", + "organization_id", ] for col in target_columns: if col not in downloaded_metadata.columns: downloaded_metadata[col] = None - if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == True: + if ( + downloaded_metadata.isna().all().all() == False + or downloaded_metadata.empty == True + ): downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda point: point.within(polygon)) + downloaded_metadata["geometry"].apply( + lambda point: point.within(polygon) + ) ] return downloaded_metadata @@ -187,9 +198,7 @@ def filter_results( df = results_df.copy() if is_pano is not None: if df["is_pano"].isna().all(): - logger.exception( - "No Mapillary Feature in the AoI has a 'is_pano' value." - ) + logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] @@ -220,14 +229,12 @@ def get_image_metadata( organization_id: str = None, start_time: str = None, end_time: str = None, - sampling_threshold = None, + sampling_threshold=None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) - downloaded_metadata = coordinate_download( - aoi_polygon, level, attempt_limit - ) + downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) downloaded_metadata = downloaded_metadata[ - downloaded_metadata['geometry'].apply(lambda geom: isinstance(geom, Point)) + downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] downloaded_metadata = filter_results( @@ -235,10 +242,15 @@ def get_image_metadata( ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - if downloaded_metadata.isna().all().all() == False or downloaded_metadata.empty == False: + if ( + downloaded_metadata.isna().all().all() == False + or downloaded_metadata.empty == False + ): if len(downloaded_metadata) > 100000: - err = (f"Too many Images with selected filter " - f"options for the AoI: {len(downloaded_metadata)}") + err = ( + f"Too many Images with selected filter " + f"options for the AoI: {len(downloaded_metadata)}" + ) raise ValueError(err) else: return { diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 082346b96..1d9c53dc0 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -3,6 +3,7 @@ from shapely import wkt from shapely.geometry import Point + def distance_on_sphere(p1, p2): """ p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat @@ -30,13 +31,19 @@ def distance_on_sphere(p1, p2): delta_lat = p2[1] - p1[1] delta_long = p2[0] - p1[0] - a = np.sin(delta_lat / 2) ** 2 + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + a = ( + np.sin(delta_lat / 2) ** 2 + + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + ) c = 2 * np.arcsin(np.sqrt(a)) distances = earth_radius * c return distances + """-----------------------------------Filtering Points------------------------------------------------""" + + def filter_points(df, threshold_distance): """ Filter points from a DataFrame based on a threshold distance. @@ -61,12 +68,10 @@ def filter_points(df, threshold_distance): lat = df["lat"].to_numpy() long = df["long"].to_numpy() - - distances = distance_on_sphere([long[1:],lat[1:]], - [long[:-1],lat[:-1]]) + distances = distance_on_sphere([long[1:], lat[1:]], [long[:-1], lat[:-1]]) road_length = np.sum(distances) - #save the last point if the road segment is relavitely small (< 2*road_length) + # save the last point if the road segment is relavitely small (< 2*road_length) if threshold_distance <= road_length < 2 * threshold_distance: mask[-1] = True @@ -74,18 +79,26 @@ def filter_points(df, threshold_distance): for i, distance in enumerate(distances): accumulated_distance += distance if accumulated_distance >= threshold_distance: - mask[i+1] = True + mask[i + 1] = True accumulated_distance = 0 # Reset accumulated distance to_be_returned_df = df[mask] # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually p2 = to_be_returned_df.iloc[0] - distance = distance_on_sphere([float(p2["long"]),float(p2["lat"])],[long[-1],lat[-1]]) - - #last point will be added if it suffices the length condition - #last point will be added in case there is only one point returned - if distance >= threshold_distance or len(to_be_returned_df) ==1: - to_be_returned_df = pd.concat([to_be_returned_df,pd.DataFrame(df.iloc[-1],columns=to_be_returned_df.columns)],axis=0) + distance = distance_on_sphere( + [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] + ) + + # last point will be added if it suffices the length condition + # last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) == 1: + to_be_returned_df = pd.concat( + [ + to_be_returned_df, + pd.DataFrame(df.iloc[-1], columns=to_be_returned_df.columns), + ], + axis=0, + ) return to_be_returned_df @@ -109,19 +122,23 @@ def spatial_sampling(df, interval_length): if len(df) == 1: return df - df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) - df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) - sorted_df = df.sort_values(by=['captured_at']) + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) + sorted_df = df.sort_values(by=["captured_at"]) sampled_sequence_df = pd.DataFrame() # loop through each sequence - for sequence in sorted_df['sequence_id'].unique(): - sequence_df = sorted_df[sorted_df['sequence_id'] == sequence] - - filtered_sorted_sub_df = filter_points(sequence_df,interval_length) - sampled_sequence_df = pd.concat([sampled_sequence_df,filtered_sorted_sub_df],axis=0) - + for sequence in sorted_df["sequence_id"].unique(): + sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] + filtered_sorted_sub_df = filter_points(sequence_df, interval_length) + sampled_sequence_df = pd.concat( + [sampled_sequence_df, filtered_sorted_sub_df], axis=0 + ) return sampled_sequence_df From 8593a3c7112ea9171f7dd63b28cb6beb80d63377 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 15:08:51 +0100 Subject: [PATCH 032/107] fix: fixed tests and removed return of failed rows for download from mapillary --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 4 ++-- mapswipe_workers/tests/unittests/test_project_type_street.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 3558e9b38..6cd77e787 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -85,13 +85,13 @@ def download_and_process_tile(row, attempt_limit=3): data = pd.DataFrame(data) if not data.empty: - return data, None + return data except Exception as e: print(f"An exception occurred while requesting a tile: {e}") attempt += 1 print(f"A tile could not be downloaded: {row}") - return None, row + return None def coordinate_download( diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py index 64b80b3e7..8ec8d0fa4 100644 --- a/mapswipe_workers/tests/unittests/test_project_type_street.py +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -34,7 +34,7 @@ def setUp(self) -> None: df = pd.read_csv(file) df['geometry'] = df['geometry'].apply(wkt.loads) - mock_get.return_value = (df, None) + mock_get.return_value = df self.project = StreetProject(project_draft) def test_init(self): From 9503a7d8c1d180f1f3a01042da55add4dffa200f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 28 Nov 2024 15:51:45 +0100 Subject: [PATCH 033/107] fix: tests for removed failing rows --- mapswipe_workers/tests/unittests/test_process_mapillary.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 8f918f0c0..a94520a67 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -173,9 +173,8 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): row = {"x": 1, "y": 1, "z": 14} - result, failed = download_and_process_tile(row) + result = download_and_process_tile(row) - self.assertIsNone(failed) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") @@ -188,10 +187,9 @@ def test_download_and_process_tile_failure(self, mock_get): mock_get.return_value = mock_response row = pd.Series({"x": 1, "y": 1, "z": self.level}) - result, failed = download_and_process_tile(row) + result = download_and_process_tile(row) self.assertIsNone(result) - self.assertIsNotNone(failed) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): From c0016e26dce13bd3ab25653a799d16420f150619 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 16:40:35 +0100 Subject: [PATCH 034/107] style: fix flake8 errors and isort --- .../project_types/street/project.py | 19 ++++---- .../utils/process_mapillary.py | 29 ++++++------ .../utils/spatial_sampling.py | 47 ++++++++++--------- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index fba87f390..e15f42c1f 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -1,11 +1,8 @@ -import json -import os -import urllib import math - -from osgeo import ogr from dataclasses import dataclass -from mapswipe_workers.definitions import DATA_PATH, logger +from typing import Dict, List + +from mapswipe_workers.definitions import logger from mapswipe_workers.firebase.firebase import Firebase from mapswipe_workers.firebase_to_postgres.transfer_results import ( results_to_file, @@ -15,16 +12,16 @@ from mapswipe_workers.generate_stats.project_stats import ( get_statistics_for_integer_result_project, ) +from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask +from mapswipe_workers.utils.process_mapillary import get_image_metadata from mapswipe_workers.utils.validate_input import ( - check_if_layer_is_empty, - load_geojson_to_ogr, build_multipolygon_from_layer_geometries, check_if_layer_has_too_many_geometries, - save_geojson_to_file, + check_if_layer_is_empty, + load_geojson_to_ogr, multipolygon_to_wkt, + save_geojson_to_file, ) -from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup -from mapswipe_workers.utils.process_mapillary import get_image_metadata @dataclass diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index c55b213d9..c17839691 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -1,23 +1,22 @@ +import os +from concurrent.futures import ThreadPoolExecutor, as_completed + import mercantile -import json +import pandas as pd import requests -import os -import time from shapely import ( - box, - Polygon, - MultiPolygon, - Point, LineString, MultiLineString, + MultiPolygon, + Point, + Polygon, + box, unary_union, ) from shapely.geometry import shape -import pandas as pd from vt2geojson import tools as vt2geojson_tools -from concurrent.futures import ThreadPoolExecutor, as_completed -from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY -from mapswipe_workers.definitions import logger + +from mapswipe_workers.definitions import MAPILLARY_API_KEY, MAPILLARY_API_LINK, logger from mapswipe_workers.utils.spatial_sampling import spatial_sampling @@ -140,8 +139,8 @@ def coordinate_download( downloaded_metadata[col] = None if ( - downloaded_metadata.isna().all().all() == False - or downloaded_metadata.empty == True + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is True ): downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply( @@ -243,8 +242,8 @@ def get_image_metadata( if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if ( - downloaded_metadata.isna().all().all() == False - or downloaded_metadata.empty == False + downloaded_metadata.isna().all().all() is False + or downloaded_metadata.empty is False ): if len(downloaded_metadata) > 100000: err = ( diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 1d9c53dc0..3cafb5362 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -1,26 +1,29 @@ import numpy as np import pandas as pd -from shapely import wkt -from shapely.geometry import Point def distance_on_sphere(p1, p2): """ - p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat - coordinates of the points in set1 and set2 + p1 and p2 are two lists that have two elements. They are numpy arrays of the long + and lat coordinates of the points in set1 and set2 - Calculate the distance between two points on the Earth's surface using the haversine formula. + Calculate the distance between two points on the Earth's surface using the + haversine formula. Args: - p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree - p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree + p1 (list): Array containing the longitude and latitude coordinates of points + FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points + TO which the distance to be calculated in degree Returns: - numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers. + numpy.ndarray: Array containing the distances between the two points on the + sphere in kilometers. - This function computes the distance between two points on the Earth's surface using the haversine formula, - which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain - longitude and latitude coordinates in degrees. The function returns an array containing the distances + This function computes the distance between two points on the Earth's surface + using the haversine formula, which takes into account the spherical shape of the + Earth. The input arrays `p1` and `p2` should contain longitude and latitude + coordinates in degrees. The function returns an array containing the distances between corresponding pairs of points. """ earth_radius = 6371 # km @@ -41,7 +44,7 @@ def distance_on_sphere(p1, p2): return distances -"""-----------------------------------Filtering Points------------------------------------------------""" +"""----------------------------Filtering Points-------------------------------""" def filter_points(df, threshold_distance): @@ -56,10 +59,11 @@ def filter_points(df, threshold_distance): pandas.DataFrame: Filtered DataFrame containing selected points. float: Total road length calculated from the selected points. - This function filters points from a DataFrame based on the given threshold distance. It calculates - distances between consecutive points and accumulates them until the accumulated distance surpasses - the threshold distance. It then selects those points and constructs a new DataFrame. Additionally, - it manually checks the last point to include it if it satisfies the length condition. The function + This function filters points from a DataFrame based on the given threshold + distance. It calculates distances between consecutive points and accumulates them + until the accumulated distance surpasses the threshold distance. It then selects + those points and constructs a new DataFrame. Additionally, it manually checks the + last point to include it if it satisfies the length condition. The function returns the filtered DataFrame along with the calculated road length. """ road_length = 0 @@ -83,7 +87,8 @@ def filter_points(df, threshold_distance): accumulated_distance = 0 # Reset accumulated distance to_be_returned_df = df[mask] - # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually + # since the last point has to be omitted in the vectorized distance calculation, + # it is being checked manually p2 = to_be_returned_df.iloc[0] distance = distance_on_sphere( [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] @@ -114,10 +119,10 @@ def spatial_sampling(df, interval_length): geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. float: Total road length calculated from the selected points. - This function calculates the spacing between points in a GeoDataFrame by filtering points - based on the provided interval length. It first sorts the GeoDataFrame by timestamp and - then filters points using the filter_points function. The function returns the filtered - GeoDataFrame along with the total road length. + This function calculates the spacing between points in a GeoDataFrame by filtering + points based on the provided interval length. It first sorts the GeoDataFrame by + timestamp and then filters points using the filter_points function. The function + returns the filtered GeoDataFrame along with the total road length. """ if len(df) == 1: return df From 2f3d523a03abf6b0e247e02cfb880200c56e89a1 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 16:47:12 +0100 Subject: [PATCH 035/107] style: isort --- mapswipe_workers/mapswipe_workers/project_types/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index 048eda71c..43013b0dc 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -2,13 +2,13 @@ from .arbitrary_geometry.footprint.project import FootprintProject from .arbitrary_geometry.footprint.tutorial import FootprintTutorial from .media_classification.project import MediaClassificationProject +from .street.project import StreetProject from .tile_map_service.change_detection.project import ChangeDetectionProject from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial from .tile_map_service.classification.project import ClassificationProject from .tile_map_service.classification.tutorial import ClassificationTutorial from .tile_map_service.completeness.project import CompletenessProject from .tile_map_service.completeness.tutorial import CompletenessTutorial -from .street.project import StreetProject __all__ = [ "ClassificationProject", From 6ad174f052fab8e289130c48fb68d90b1c0b8fc2 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 17:17:47 +0100 Subject: [PATCH 036/107] build: add requirements --- mapswipe_workers/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 7e125fe4b..588754060 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -14,3 +14,6 @@ sentry-sdk==0.18.0 six==1.15.0 slackclient==2.9.2 xdg==4.0.1 +shapely +mercantile +vt2geojson From 315d239dc61340f92b0a1c0144d57887851a3ba0 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 17:28:13 +0100 Subject: [PATCH 037/107] build: add dummy mapillary key to github workflow --- .github/workflows/actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index aacc4dcd3..d66de00b8 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,6 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key + MAPILLARY_API_KEY: test-mapillary-api-key COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ From 2b88cf7f491c9088cb641989b0cc8efa5b006590 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 18:15:07 +0100 Subject: [PATCH 038/107] fix: use os.getenv instead of os.environ --- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index c9dec6d79..b6be718ef 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] +MAPILLARY_API_KEY = os.getenv("MAPILLARY_API_KEY") # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From 24c7eec528c38a22ad7f1705697fd0aacb1b50fb Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 28 Nov 2024 18:44:41 +0100 Subject: [PATCH 039/107] test: rename class and correct comments --- .../tests/integration/test_create_street_project.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py index ecf97c626..fd0608f98 100644 --- a/mapswipe_workers/tests/integration/test_create_street_project.py +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -8,7 +8,7 @@ from tests.integration import set_up, tear_down -class TestCreateFootprintProject(unittest.TestCase): +class TestCreateStreetProject(unittest.TestCase): def setUp(self): self.project_id = [ set_up.create_test_project_draft("street", "street"), @@ -33,7 +33,7 @@ def test_create_street_project(self): result = pg_db.retr_query(query, [element])[0][0] self.assertEqual(result, element) - # check if usernames made it to postgres + # check if tasks made it to postgres query = """ SELECT count(*) FROM tasks @@ -51,10 +51,11 @@ def test_create_street_project(self): result = ref.get(shallow=True) self.assertIsNotNone(result) - # Footprint projects have tasks in Firebase + # Street projects have tasks in Firebase ref = fb_db.reference(f"/v2/tasks/{element}") result = ref.get(shallow=True) self.assertIsNotNone(result) + if __name__ == "__main__": unittest.main() From 7f46e80d875eb39b49f77d70c6b71ca77c61b9bc Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 13:20:25 +0100 Subject: [PATCH 040/107] fix: remove left overs of failed rows --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index c55b213d9..62dd00d5a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -100,7 +100,6 @@ def coordinate_download( tiles = create_tiles(polygon, level) downloaded_metadata = [] - failed_tiles = [] if not tiles.empty: if not use_concurrency: @@ -115,12 +114,10 @@ def coordinate_download( for future in as_completed(futures): if future is not None: - df, failed_row = future.result() + df = future.result() if df is not None and not df.empty: downloaded_metadata.append(df) - if failed_row is not None: - failed_tiles.append(failed_row) if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: From bfa0818cd8cfb367c984ba2093e7d990774a27f4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 13:37:51 +0100 Subject: [PATCH 041/107] fix: unittests for tile download --- .../tests/unittests/test_process_mapillary.py | 62 ++++++++----------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index a94520a67..8134ad2bf 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -1,24 +1,19 @@ -import unittest -import os import json -from shapely.geometry import ( - Polygon, - MultiPolygon, - Point, - LineString, - MultiLineString, - GeometryCollection, -) -from shapely import wkt +import os +import unittest +from unittest.mock import MagicMock, patch + import pandas as pd -from unittest.mock import patch, MagicMock +from shapely import wkt +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon + from mapswipe_workers.utils.process_mapillary import ( + coordinate_download, create_tiles, download_and_process_tile, - coordinate_download, - geojson_to_polygon, filter_by_timerange, filter_results, + geojson_to_polygon, get_image_metadata, ) @@ -47,7 +42,7 @@ def setUpClass(cls): "r", ) as file: df = pd.read_csv(file) - df['geometry'] = df['geometry'].apply(wkt.loads) + df["geometry"] = df["geometry"].apply(wkt.loads) cls.fixture_df = df def setUp(self): @@ -141,7 +136,10 @@ def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): } with self.assertRaises(ValueError) as context: geojson_to_polygon(geojson_data) - self.assertEqual(str(context.exception), "Non-polygon geometries cannot be combined into a MultiPolygon.") + self.assertEqual( + str(context.exception), + "Non-polygon geometries cannot be combined into a MultiPolygon.", + ) def test_geojson_to_polygon_empty_feature_collection(self): geojson_data = {"type": "FeatureCollection", "features": []} @@ -193,27 +191,17 @@ def test_download_and_process_tile_failure(self, mock_get): @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = ( - pd.DataFrame([{"geometry": None}]), - None, - ) + mock_download_and_process_tile.return_value = pd.DataFrame([{"geometry": None}]) - metadata = coordinate_download( - self.test_polygon, self.level - ) + metadata = coordinate_download(self.test_polygon, self.level) self.assertIsInstance(metadata, pd.DataFrame) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = ( - None, - pd.Series({"x": 1, "y": 1, "z": self.level}), - ) + mock_download_and_process_tile.return_value = pd.DataFrame() - metadata = coordinate_download( - self.test_polygon, self.level - ) + metadata = coordinate_download(self.test_polygon, self.level) self.assertTrue(metadata.empty) @@ -268,7 +256,11 @@ def test_filter_no_rows_after_filter(self): self.assertTrue(filtered_df.empty) def test_filter_missing_columns(self): - columns_to_check = ["is_pano", "organization_id", "captured_at"] # Add your column names here + columns_to_check = [ + "is_pano", + "organization_id", + "captured_at", + ] # Add your column names here for column in columns_to_check: df_copy = self.fixture_df.copy() df_copy[column] = None @@ -286,7 +278,6 @@ def test_get_image_metadata(self, mock_coordinate_download): self.assertIn("ids", result) self.assertIn("geometries", result) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_filtering(self, mock_coordinate_download): mock_coordinate_download.return_value = self.fixture_df @@ -302,7 +293,6 @@ def test_get_image_metadata_filtering(self, mock_coordinate_download): self.assertIn("ids", result) self.assertIn("geometries", result) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_no_rows(self, mock_coordinate_download): mock_coordinate_download.return_value = self.fixture_df @@ -326,8 +316,10 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.filter_results") @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") - def test_get_image_metadata_size_restriction(self, mock_coordinate_download, mock_filter_results): - mock_filter_results.return_value = pd.DataFrame({'ID': range(1, 100002)}) + def test_get_image_metadata_size_restriction( + self, mock_coordinate_download, mock_filter_results + ): + mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) mock_coordinate_download.return_value = self.fixture_df with self.assertRaises(ValueError): From 816b1e6f48697e5c2b6ad81c7ef8a74e5dd51f5d Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 14:54:33 +0100 Subject: [PATCH 042/107] fix: add condition to raise valueerror if no features are found in aoi --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index fa2922f93..4b6943fdf 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -229,6 +229,8 @@ def get_image_metadata( ): aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) + if downloaded_metadata.isna().all().all() or downloaded_metadata.empty: + raise ValueError("No Mapillary Features in the AoI.") downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] From f287b3ae138f381392b289baab5c939666aa3984 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 15:16:27 +0100 Subject: [PATCH 043/107] fix: use mapillary api key secret in github workflow --- .github/workflows/actions.yml | 2 +- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index d66de00b8..4609eb697 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,7 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key - MAPILLARY_API_KEY: test-mapillary-api-key + MAPILLARY_API_KEY: ${{ secrets.MAPILLARY_API_KEY }} COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index b6be718ef..e5037675b 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.getenv("MAPILLARY_API_KEY") +MAPILLARY_API_KEY = os.environ("MAPILLARY_API_KEY") # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From 020178d6e57cac481c000af831f2265647944b7d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 15:29:44 +0100 Subject: [PATCH 044/107] fix: use square brackets with os.environ --- mapswipe_workers/mapswipe_workers/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index e5037675b..c9dec6d79 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -17,7 +17,7 @@ OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" -MAPILLARY_API_KEY = os.environ("MAPILLARY_API_KEY") +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 From dcaab3d045b62191bc847ebf1e8c4bc47c45ea44 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 16:28:29 +0100 Subject: [PATCH 045/107] fix: add mapillary key to docker compose --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index ccbc2aa9d..d465a704c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,7 @@ x-mapswipe-workers: &base_mapswipe_workers SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' OSMCHA_API_KEY: '${OSMCHA_API_KEY}' + MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}' depends_on: - postgres volumes: From 81284889ca7c465787f32621b51d930c3e927330 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 3 Dec 2024 17:02:12 +0100 Subject: [PATCH 046/107] fix: allow tasks with point geom in postgres --- postgres/initdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 74064f835999db1fbd86155975fbac5ad3575f0b Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:15:17 +0100 Subject: [PATCH 047/107] fix: change geometry type in tasks table in initdb --- postgres/initdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 729a4be11567c2cf82c9068d8d2f6f37c5b264cd Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:29:07 +0100 Subject: [PATCH 048/107] fix: change to all geometries in tasks table in setup db --- mapswipe_workers/tests/integration/set_up_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index ce9b97197..b03e76699 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(GEOMETRY, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 994f3983157a8a0ee980cdb61a737e38f12060c3 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 3 Dec 2024 17:44:36 +0100 Subject: [PATCH 049/107] fix: use lower case in setup db --- mapswipe_workers/tests/integration/set_up_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index b03e76699..f954d3a8c 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(GEOMETRY, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), From 1c1a6ba3f934f67735aa234f83358ed462689816 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 4 Dec 2024 15:16:10 +0100 Subject: [PATCH 050/107] feat: add integration test for tutorials --- mapswipe_workers/tests/integration/set_up.py | 55 ++++++++++---- .../tests/integration/test_create_tutorial.py | 71 +++++++++++++++++++ 2 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 mapswipe_workers/tests/integration/test_create_tutorial.py diff --git a/mapswipe_workers/tests/integration/set_up.py b/mapswipe_workers/tests/integration/set_up.py index 74adc6fda..1c3c0bdf4 100644 --- a/mapswipe_workers/tests/integration/set_up.py +++ b/mapswipe_workers/tests/integration/set_up.py @@ -16,20 +16,28 @@ def set_firebase_test_data( - project_type: str, data_type: str, fixture_name: str, identifier: str + project_type: str, + data_type: str, + fixture_name: str, + identifier: str, + tutorial_id: str = None, ): test_dir = os.path.dirname(__file__) fixture_name = fixture_name + ".json" file_path = os.path.join( test_dir, "fixtures", project_type, data_type, fixture_name ) - upload_file_to_firebase(file_path, data_type, identifier) + upload_file_to_firebase(file_path, data_type, identifier, tutorial_id=tutorial_id) -def upload_file_to_firebase(file_path: str, data_type: str, identifier: str): +def upload_file_to_firebase( + file_path: str, data_type: str, identifier: str, tutorial_id: str = None +): with open(file_path) as test_file: test_data = json.load(test_file) + if tutorial_id: + test_data["tutorialId"] = tutorial_id fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/{data_type}/{identifier}") ref.set(test_data) @@ -85,15 +93,20 @@ def create_test_project( set_postgres_test_data(project_type, "users", "user") set_firebase_test_data(project_type, "user_groups", "user_group", "") set_firebase_test_data(project_type, "results", fixture_name, project_id) - set_postgres_test_data(project_type, "mapping_sessions", fixture_name, columns=[ - "project_id", - "group_id", - "user_id", - "mapping_session_id", - "start_time", - "end_time", - "items_count", - ]) + set_postgres_test_data( + project_type, + "mapping_sessions", + fixture_name, + columns=[ + "project_id", + "group_id", + "user_id", + "mapping_session_id", + "start_time", + "end_time", + "items_count", + ], + ) set_postgres_test_data(project_type, mapping_sessions_results, fixture_name) if create_user_group_session_data: set_postgres_test_data( @@ -108,7 +121,9 @@ def create_test_project( "created_at", ], ) - set_postgres_test_data(project_type, "mapping_sessions_user_groups", fixture_name) + set_postgres_test_data( + project_type, "mapping_sessions_user_groups", fixture_name + ) time.sleep(5) # Wait for Firebase Functions to complete return project_id @@ -131,12 +146,24 @@ def create_test_user(project_type: str, user_id: str = None) -> str: def create_test_project_draft( - project_type: str, fixture_name: str = "user", identifier: str = "" + project_type: str, + fixture_name: str = "user", + identifier: str = "", + tutorial_id: str = None, ) -> str: """ Create test project drafts in Firebase and return project ids. Project drafts in Firebase are created by project manager using the dashboard. """ + if tutorial_id: + set_firebase_test_data( + project_type, + "projectDrafts", + fixture_name, + identifier, + tutorial_id=tutorial_id, + ) + return identifier if not identifier: identifier = f"test_{fixture_name}" set_firebase_test_data(project_type, "projectDrafts", fixture_name, identifier) diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py new file mode 100644 index 000000000..76327120b --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -0,0 +1,71 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateTileClassificationProject(unittest.TestCase): + def setUp(self): + self.tutorial_id = set_up.create_test_tutorial_draft("footprint", "footprint") + + self.project_id = set_up.create_test_project_draft( + "tile_classification", + "tile_classification", + "test_tile_classification_tutorial", + tutorial_id=self.tutorial_id, + ) + create_directories() + + def tearDown(self): + tear_down.delete_test_data(self.project_id) + + def test_create_tile_classification_project(self): + runner = CliRunner() + runner.invoke(mapswipe_workers.run_create_projects, catch_exceptions=False) + + pg_db = auth.postgresDB() + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [self.project_id])[0][0] + self.assertEqual(result, self.project_id) + + query = """ + SELECT project_id + FROM projects + WHERE project_id = %s + and project_type_specifics::jsonb ? 'customOptions' + """ + result = pg_db.retr_query(query, [self.project_id])[0][0] + self.assertEqual(result, self.project_id) + + query = "SELECT count(*) FROM groups WHERE project_id = %s" + result = pg_db.retr_query(query, [self.project_id])[0][0] + self.assertEqual(result, 20) + + query = "SELECT count(*) FROM tasks WHERE project_id = %s" + result = pg_db.retr_query(query, [self.project_id])[0][0] + self.assertEqual(result, 5040) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{self.project_id}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{self.project_id}") + result = ref.get(shallow=True) + self.assertEqual(len(result), 20) + + # Tile classification projects do not have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{self.project_id}") + result = ref.get(shallow=True) + self.assertIsNone(result) + + ref = fb_db.reference(f"/v2/projects/{self.project_id}/tutorialId") + result = ref.get(shallow=True) + self.assertEqual(self.tutorial_id, result) + + +if __name__ == "__main__": + unittest.main() From dadca061437099800b41f9f4af1e3d56ad5ba218 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Wed, 4 Dec 2024 15:43:33 +0100 Subject: [PATCH 051/107] fix: delete tutorial draft at end of test --- .../tests/integration/test_create_tutorial.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py index 76327120b..c0904f06b 100644 --- a/mapswipe_workers/tests/integration/test_create_tutorial.py +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -9,7 +9,11 @@ class TestCreateTileClassificationProject(unittest.TestCase): def setUp(self): - self.tutorial_id = set_up.create_test_tutorial_draft("footprint", "footprint") + self.tutorial_id = set_up.create_test_tutorial_draft( + "tile_classification", + "tile_classification", + "test_tile_classification_tutorial", + ) self.project_id = set_up.create_test_project_draft( "tile_classification", @@ -66,6 +70,8 @@ def test_create_tile_classification_project(self): result = ref.get(shallow=True) self.assertEqual(self.tutorial_id, result) + breakpoint() + if __name__ == "__main__": unittest.main() From e1ec359e3ee74d15d78c4b6a8f739e3d395b201f Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 10 Dec 2024 14:46:04 +0100 Subject: [PATCH 052/107] feat(manager-dashboard): street project definitions --- manager-dashboard/app/Base/configs/projectTypes.ts | 2 ++ manager-dashboard/app/utils/common.tsx | 4 +++- manager-dashboard/app/views/NewProject/index.tsx | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/manager-dashboard/app/Base/configs/projectTypes.ts b/manager-dashboard/app/Base/configs/projectTypes.ts index fd8d2df8e..e2f7f74eb 100644 --- a/manager-dashboard/app/Base/configs/projectTypes.ts +++ b/manager-dashboard/app/Base/configs/projectTypes.ts @@ -3,6 +3,7 @@ import { PROJECT_TYPE_BUILD_AREA, PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, + PROJECT_TYPE_STREET, PROJECT_TYPE_COMPLETENESS, } from '#utils/common'; @@ -15,6 +16,7 @@ const mapswipeProjectTypeOptions: { { value: PROJECT_TYPE_BUILD_AREA, label: 'Find' }, { value: PROJECT_TYPE_FOOTPRINT, label: 'Validate' }, { value: PROJECT_TYPE_CHANGE_DETECTION, label: 'Compare' }, + { value: PROJECT_TYPE_STREET, label: 'Street' }, { value: PROJECT_TYPE_COMPLETENESS, label: 'Completeness' }, ]; diff --git a/manager-dashboard/app/utils/common.tsx b/manager-dashboard/app/utils/common.tsx index 571be709c..1a37b3e8c 100644 --- a/manager-dashboard/app/utils/common.tsx +++ b/manager-dashboard/app/utils/common.tsx @@ -65,8 +65,9 @@ export const PROJECT_TYPE_BUILD_AREA = 1; export const PROJECT_TYPE_FOOTPRINT = 2; export const PROJECT_TYPE_CHANGE_DETECTION = 3; export const PROJECT_TYPE_COMPLETENESS = 4; +export const PROJECT_TYPE_STREET = 7; -export type ProjectType = 1 | 2 | 3 | 4; +export type ProjectType = 1 | 2 | 3 | 4 | 7; export const projectTypeLabelMap: { [key in ProjectType]: string @@ -75,6 +76,7 @@ export const projectTypeLabelMap: { [PROJECT_TYPE_FOOTPRINT]: 'Validate', [PROJECT_TYPE_CHANGE_DETECTION]: 'Compare', [PROJECT_TYPE_COMPLETENESS]: 'Completeness', + [PROJECT_TYPE_STREET]: 'Street', }; export type IconKey = 'add-outline' diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index f63bbc065..6456527da 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -59,6 +59,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_CHANGE_DETECTION, + PROJECT_TYPE_STREET, formatProjectTopic, } from '#utils/common'; import { getValueFromFirebase } from '#utils/firebase'; From 0188ba128ac7996d5519828a7c82376b6aa952ba Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 10 Dec 2024 14:55:55 +0100 Subject: [PATCH 053/107] feat(manager-dashboard): invisible tile server input in street projects --- .../app/views/NewProject/index.tsx | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 6456527da..8f9329552 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -418,6 +418,11 @@ function NewProject(props: Props) { || projectSubmissionStatus === 'projectSubmit' ); + const tileServerVisible = value.projectType === PROJECT_TYPE_BUILD_AREA + || value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_CHANGE_DETECTION; + const tileServerBVisible = value.projectType === PROJECT_TYPE_CHANGE_DETECTION || value.projectType === PROJECT_TYPE_COMPLETENESS; @@ -662,17 +667,19 @@ function NewProject(props: Props) { /> - - - + {tileServerVisible && ( + + + + )} {tileServerBVisible && ( Date: Tue, 10 Dec 2024 15:26:41 +0100 Subject: [PATCH 054/107] feat(manager-dashboard): add custom options preview and project aoi geometry input to street --- manager-dashboard/app/views/NewProject/index.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 8f9329552..e1d86a2dc 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -474,7 +474,8 @@ function NewProject(props: Props) { /> {( - value.projectType === PROJECT_TYPE_FOOTPRINT + (value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_STREET) && customOptions && customOptions.length > 0 ) && ( @@ -532,7 +533,8 @@ function NewProject(props: Props) { )} {(value.projectType === PROJECT_TYPE_BUILD_AREA || value.projectType === PROJECT_TYPE_CHANGE_DETECTION - || value.projectType === PROJECT_TYPE_COMPLETENESS) && ( + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_STREET) && ( From c4f43c99282aeb6e3020231bc714727768f59387 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 10 Dec 2024 15:42:59 +0100 Subject: [PATCH 055/107] feat(manager-dashboard): set default groupSize for street --- manager-dashboard/app/views/NewProject/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index e607d3a15..494411472 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -34,6 +34,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, + PROJECT_TYPE_STREET, IconKey, } from '#utils/common'; @@ -541,7 +542,9 @@ export function getGroupSize(projectType: ProjectType | undefined) { return 120; } - if (projectType === PROJECT_TYPE_FOOTPRINT || projectType === PROJECT_TYPE_CHANGE_DETECTION) { + if (projectType === PROJECT_TYPE_FOOTPRINT + || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_STREET) { return 25; } From f9ae7689f63f2c594bf271271c8e6aa5979fabf4 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 10 Dec 2024 16:43:11 +0100 Subject: [PATCH 056/107] feat(manager-dashboard): add banner to alert that street projects are web only --- manager-dashboard/app/views/NewProject/index.tsx | 11 +++++++++++ manager-dashboard/app/views/NewProject/styles.css | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index e1d86a2dc..065ed63bd 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -50,6 +50,7 @@ import Button from '#components/Button'; import NonFieldError from '#components/NonFieldError'; import AnimatedSwipeIcon from '#components/AnimatedSwipeIcon'; import ExpandableContainer from '#components/ExpandableContainer'; +import AlertBanner from '#components/AlertBanner'; import { valueSelector, labelSelector, @@ -465,6 +466,16 @@ function NewProject(props: Props) { error={error?.projectType} disabled={submissionPending || testPending} /> + {value.projectType === PROJECT_TYPE_STREET && ( + +
+
+ Projects of this type are currently + only visible in the web app. +
+
+ + )} Date: Tue, 10 Dec 2024 17:06:15 +0100 Subject: [PATCH 057/107] feat(manager-dashboard): add street specific inputs (wip) --- .../app/views/NewProject/index.tsx | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 065ed63bd..ba40ff364 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -707,6 +707,30 @@ function NewProject(props: Props) { /> )} + + {value.projectType === PROJECT_TYPE_STREET && ( + /* TODO: Add street project inputs for + startTimestamp, + endTimeStamp, + isPano, + organizationId, + samplingThreshold + */ + + + + )} + {error?.[nonFieldError] && (
{error?.[nonFieldError]} From 57dc509ee472a7db7c75ac5abf720db5a60704eb Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 09:09:40 +0100 Subject: [PATCH 058/107] feat(django): add new project types to project type model --- django/apps/existing_database/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index ef65496b0..06fc81190 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -1,6 +1,7 @@ +from mapswipe.db import Model + from django.contrib.gis.db import models as gis_models from django.db import models -from mapswipe.db import Model # NOTE: Django model defination and existing table structure doesn't entirely matches. # This is to be used for testing only. @@ -66,6 +67,9 @@ class Type(models.IntegerChoices): FOOTPRINT = 2, "Validate" CHANGE_DETECTION = 3, "Compare" COMPLETENESS = 4, "Completeness" + MEDIA = 5, "Media" + DIGITIZATION = 6, "Digitization" + STREET = 7, "Street" project_id = models.CharField(primary_key=True, max_length=999) created = models.DateTimeField(blank=True, null=True) From 26c822c962cdf104b17f7edd154ddb9a6acad5b3 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 09:56:12 +0100 Subject: [PATCH 059/107] fix: sort imports --- django/apps/existing_database/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 06fc81190..5bc85e113 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -1,7 +1,6 @@ -from mapswipe.db import Model - from django.contrib.gis.db import models as gis_models from django.db import models +from mapswipe.db import Model # NOTE: Django model defination and existing table structure doesn't entirely matches. # This is to be used for testing only. From 4787dd4025ce6648261e90f234ae269aeac1b145 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 10:18:05 +0100 Subject: [PATCH 060/107] fix: add new project types to schema.graphql --- django/schema.graphql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django/schema.graphql b/django/schema.graphql index d0187f7f1..b5596fc46 100644 --- a/django/schema.graphql +++ b/django/schema.graphql @@ -97,6 +97,9 @@ enum ProjectTypeEnum { FOOTPRINT CHANGE_DETECTION COMPLETENESS + MEDIA + DIGITIZATION + STREET } type ProjectTypeSwipeStatsType { From 78c4908673831f16303723349afe7290f290ce08 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 13:43:58 +0100 Subject: [PATCH 061/107] feat(manager-dashboard): add validation for organizationId --- manager-dashboard/app/views/NewProject/utils.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index 494411472..823e9a324 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -76,6 +76,7 @@ export interface ProjectFormType { tileServer: TileServer; tileServerB?: TileServer; customOptions?: CustomOptionsForProject; + organizationId?: number; } export const PROJECT_INPUT_TYPE_UPLOAD = 'aoi_file'; @@ -273,6 +274,12 @@ export const projectFormSchema: ProjectFormSchema = { greaterThanCondition(0), ], }, + organizationId: { + validations: [ + integerCondition, + greaterThanCondition(0), + ], + }, }; baseSchema = addCondition( From 905dcde5dc97eed88094f10cc5df9c171115d262 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 11 Dec 2024 18:58:11 +0100 Subject: [PATCH 062/107] feat(manager-dashboard): add mapillary image filter inputs (wip) --- .../app/views/NewProject/index.tsx | 23 ++++++++++++++++++- .../app/views/NewProject/utils.ts | 7 ++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index ba40ff364..7bad14c40 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -51,6 +51,7 @@ import NonFieldError from '#components/NonFieldError'; import AnimatedSwipeIcon from '#components/AnimatedSwipeIcon'; import ExpandableContainer from '#components/ExpandableContainer'; import AlertBanner from '#components/AlertBanner'; +import Checkbox from '#components/Checkbox'; import { valueSelector, labelSelector, @@ -106,6 +107,7 @@ const defaultProjectFormValue: PartialProjectFormType = { // maxTasksPerUser: -1, inputType: PROJECT_INPUT_TYPE_UPLOAD, filter: FILTER_BUILDINGS, + isPano: false, }; interface Props { @@ -719,15 +721,34 @@ function NewProject(props: Props) { +
time range input
+
+ + +
)} diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index 823e9a324..90257d081 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -77,6 +77,8 @@ export interface ProjectFormType { tileServerB?: TileServer; customOptions?: CustomOptionsForProject; organizationId?: number; + isPano?: boolean; + samplingThreshold?: number; } export const PROJECT_INPUT_TYPE_UPLOAD = 'aoi_file'; @@ -280,6 +282,11 @@ export const projectFormSchema: ProjectFormSchema = { greaterThanCondition(0), ], }, + samplingThreshold: { + validation: [ + greaterThanCondition(0), + ], + }, }; baseSchema = addCondition( From 8c9d77402d6f4543af54ecb744680bd65ce97202 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 12 Dec 2024 14:49:57 +0100 Subject: [PATCH 063/107] WIP: add tutorial for street project --- .../mapswipe_workers/definitions.py | 2 + .../project_types/__init__.py | 2 + .../project_types/street/tutorial.py | 80 ++++++++- .../tests/fixtures/projectDrafts/street.json | 1 - .../tests/fixtures/tutorialDrafts/street.json | 170 ++++++++++++++++++ .../integration/test_create_street_project.py | 2 + .../tests/integration/test_create_tutorial.py | 4 +- .../tests/unittests/test_tutorial.py | 11 +- 8 files changed, 259 insertions(+), 13 deletions(-) create mode 100644 mapswipe_workers/tests/fixtures/tutorialDrafts/street.json diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index c9dec6d79..aa32d3aac 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -170,6 +170,7 @@ def tutorial(self): ClassificationTutorial, CompletenessTutorial, FootprintTutorial, + StreetTutorial, ) project_type_classes = { @@ -177,5 +178,6 @@ def tutorial(self): 2: FootprintTutorial, 3: ChangeDetectionTutorial, 4: CompletenessTutorial, + 7: StreetTutorial, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index 43013b0dc..9560c76ef 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -3,6 +3,7 @@ from .arbitrary_geometry.footprint.tutorial import FootprintTutorial from .media_classification.project import MediaClassificationProject from .street.project import StreetProject +from .street.tutorial import StreetTutorial from .tile_map_service.change_detection.project import ChangeDetectionProject from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial from .tile_map_service.classification.project import ClassificationProject @@ -22,4 +23,5 @@ "FootprintTutorial", "DigitizationProject", "StreetProject", + "StreetTutorial", ] diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py index cfbfc0ead..ca2c56cbe 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -1,14 +1,84 @@ +from dataclasses import asdict, dataclass + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.project_types.street.project import StreetGroup, StreetTask from mapswipe_workers.project_types.tutorial import BaseTutorial +@dataclass +class StreetTutorialTask(StreetTask): + projectId: int + taskId: str + groupId: int + referenceAnswer: int + screen: int + + class StreetTutorial(BaseTutorial): - """The subclass for an TMS Grid based Tutorial.""" + """The subclass for an arbitrary geometry based Tutorial.""" - def save_tutorial(self): - raise NotImplementedError("Currently Street has no Tutorial") + def __init__(self, tutorial_draft): + # this will create the basis attributes + super().__init__(tutorial_draft) + + # self.projectId = tutorial_draft["projectId"] + self.projectType = tutorial_draft["projectType"] + self.tutorial_tasks = tutorial_draft["tasks"] + self.groups = dict() + self.tasks = dict() def create_tutorial_groups(self): - raise NotImplementedError("Currently Street has no Tutorial") + """Create group for the tutorial based on provided examples in geojson file.""" + # load examples/tasks from file + + group = StreetGroup( + groupId=101, + projectId=self.projectId, + numberOfTasks=len(self.tutorial_tasks), + progress=0, + finishedCount=0, + requiredCount=0, + ) + self.groups[101] = group + + # Add number of tasks for the group here. This needs to be set according to + # the number of features/examples in the geojson file + + logger.info( + f"{self.projectId}" + f" - create_tutorial_groups - " + f"created groups dictionary" + ) def create_tutorial_tasks(self): - raise NotImplementedError("Currently Street has no Tutorial") + """Create the tasks dict based on provided examples in geojson file.""" + task_list = [] + for i, task in enumerate(self.tutorial_tasks): + task = StreetTutorialTask( + projectId=self.projectId, + groupId=101, + taskId=f"{task['taskImageId']}", + geometry="", + referenceAnswer=task["referenceAnswer"], + screen=i, + ) + task_list.append(asdict(task)) + if task_list: + self.tasks[101] = task_list + else: + logger.info(f"group in project {self.projectId} is not valid.") + + logger.info( + f"{self.projectId}" + f" - create_tutorial_tasks - " + f"created tasks dictionary" + ) + + def save_tutorial(self): + firebase = Firebase() + firebase.save_tutorial_to_firebase( + self, self.groups, self.tasks, useCompression=True + ) + logger.info(self.tutorialDraftId) + firebase.drop_tutorial_draft(self.tutorialDraftId) diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json index 1dd5b452a..67d1d8b04 100644 --- a/mapswipe_workers/tests/fixtures/projectDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -46,6 +46,5 @@ "requestingOrganisation": "test", "verificationNumber": 3, "groupSize": 25, - "startTimestamp": "2019-07-01T00:00:00.000Z", "samplingThreshold": 0.1 } diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json new file mode 100644 index 000000000..22116363b --- /dev/null +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json @@ -0,0 +1,170 @@ +{ + "createdBy": "LtCUyou6CnSSc1H0Q0nDrN97x892", + "tutorialDraftId": "waste_mapping_dar_es_salaam", + "taskImageIds": [ + 888464808378923, + 1552821322020271, + 2969692853315413, + 1036040467918366, + 837497816845037 + ], + "answers": [ + 1, + 2, + 1, + 2, + 3 + ], + "customOptions": [ + { + "description": "the shape does outline a building in the image", + "icon": "hand-right-outline", + "iconColor": "#00796B", + "subOptions": [ + { + "description": "doppelt", + "value": 2 + }, + { + "description": "dreifach", + "value": 3 + } + ], + "title": "Jetzt rede ich", + "value": 1 + }, + { + "description": "the shape doesn't match a building in the image", + "icon": "close-outline", + "iconColor": "#D32F2F", + "title": "No", + "value": 0 + } + ], + "informationPages": [ + { + "blocks": [ + { + "blockNumber": 1, + "blockType": "text", + "textDescription": "asdf" + }, + { + "blockNumber": 2, + "blockType": "image", + "image": "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/tutorialImages%2F1705402528654-block-image-2-base-query-form.png?alt=media&token=54325ab8-c5e7-45a3-be41-1926a5984a05" + } + ], + "pageNumber": 1, + "title": "asdf" + } + ], + "lookFor": "waste", + "name": "Waste Mapping Dar es Salaam", + "projectType": 7, + "screens": [ + null, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first areas with buildings!" + } + }, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first" + } + }, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first areas with buildings!" + } + }, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first areas with buildings!" + } + }, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first areas with buildings!" + } + }, + { + "hint": { + "description": "Swipe to learn some more.", + "icon": "swipe-left", + "title": "We've marked the correct square green." + }, + "instructions": { + "description": "Tap once to mark the squares with buildings green.", + "icon": "tap-1", + "title": "There are some buildings in this image" + }, + "success": { + "description": "Swipe to the next screen to look for more.", + "icon": "check", + "title": "You found your first areas with buildings!" + } + } + ] +} \ No newline at end of file diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py index fd0608f98..10da2bd0e 100644 --- a/mapswipe_workers/tests/integration/test_create_street_project.py +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -56,6 +56,8 @@ def test_create_street_project(self): result = ref.get(shallow=True) self.assertIsNotNone(result) + breakpoint() + if __name__ == "__main__": unittest.main() diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py index c0904f06b..5ae13c76d 100644 --- a/mapswipe_workers/tests/integration/test_create_tutorial.py +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -10,8 +10,8 @@ class TestCreateTileClassificationProject(unittest.TestCase): def setUp(self): self.tutorial_id = set_up.create_test_tutorial_draft( - "tile_classification", - "tile_classification", + "street", + "street", "test_tile_classification_tutorial", ) diff --git a/mapswipe_workers/tests/unittests/test_tutorial.py b/mapswipe_workers/tests/unittests/test_tutorial.py index 5ba1c209a..51a4cbf84 100644 --- a/mapswipe_workers/tests/unittests/test_tutorial.py +++ b/mapswipe_workers/tests/unittests/test_tutorial.py @@ -1,26 +1,27 @@ import os import unittest -from mapswipe_workers.project_types import ClassificationTutorial +from mapswipe_workers.project_types import StreetTutorial from tests.fixtures import FIXTURE_DIR, get_fixture class TestTutorial(unittest.TestCase): def test_init_tile_classification_project(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) - self.assertIsNotNone(ClassificationTutorial(tutorial_draft=tutorial_draft)) + self.assertIsNotNone(StreetTutorial(tutorial_draft=tutorial_draft)) def test_create_tile_classification_tasks(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) - tutorial = ClassificationTutorial(tutorial_draft=tutorial_draft) + tutorial = StreetTutorial(tutorial_draft=tutorial_draft) tutorial.create_tutorial_groups() tutorial.create_tutorial_tasks() self.assertTrue(tutorial.groups) self.assertTrue(tutorial.tasks) + breakpoint() if __name__ == "__main__": From e90ebe543823bffea3f1986e64a54ac3558fb69a Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 12 Dec 2024 19:54:27 +0100 Subject: [PATCH 064/107] feat(manager-dashboard): complete mapillary image filter inputs --- manager-dashboard/app/Base/styles.css | 7 +- .../Calendar/CalendarDate/index.tsx | 68 +++ .../Calendar/CalendarDate/styles.css | 25 ++ .../app/components/Calendar/index.tsx | 292 ++++++++++++ .../app/components/Calendar/styles.css | 93 ++++ .../app/components/DateRangeInput/index.tsx | 420 ++++++++++++++++++ .../DateRangeInput/predefinedDateRange.ts | 181 ++++++++ .../app/components/DateRangeInput/styles.css | 85 ++++ manager-dashboard/app/utils/common.tsx | 14 + .../app/views/NewProject/index.tsx | 21 +- .../app/views/NewProject/utils.ts | 14 + 11 files changed, 1211 insertions(+), 9 deletions(-) create mode 100644 manager-dashboard/app/components/Calendar/CalendarDate/index.tsx create mode 100644 manager-dashboard/app/components/Calendar/CalendarDate/styles.css create mode 100644 manager-dashboard/app/components/Calendar/index.tsx create mode 100644 manager-dashboard/app/components/Calendar/styles.css create mode 100644 manager-dashboard/app/components/DateRangeInput/index.tsx create mode 100644 manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts create mode 100644 manager-dashboard/app/components/DateRangeInput/styles.css diff --git a/manager-dashboard/app/Base/styles.css b/manager-dashboard/app/Base/styles.css index 306f01caf..c746dc570 100644 --- a/manager-dashboard/app/Base/styles.css +++ b/manager-dashboard/app/Base/styles.css @@ -122,9 +122,14 @@ p { --line-height-relaxed: 1.625; --line-height-loose: 2; - --shadow-card: 0 2px 4px -2px var(--color-shadow); --duration-transition-medium: .2s; + --color-background-hover-light: rgba(0, 0, 0, .04); + --width-calendar-date: 2.4rem; + + --opacity-watermark: 0.3; + --color-text-disabled: rgba(0, 0, 0, .3); + } diff --git a/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx new file mode 100644 index 000000000..2b1fed1a0 --- /dev/null +++ b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { _cs } from '@togglecorp/fujs'; + +import RawButton, { Props as RawButtonProps } from '../../RawButton'; +import { ymdToDateString, typedMemo } from '../../../utils/common.tsx'; + +import styles from './styles.css'; + +export interface Props { + className?: string; + year: number; + month: number; + date: number; + currentYear: number; + currentMonth: number; + activeDate?: string; + currentDate: number; + onClick?: (year: number, month: number, date: number) => void; + elementRef?: RawButtonProps['elementRef']; + ghost?: boolean; +} + +function CalendarDate(props: Props) { + const { + className, + year, + month, + date, + currentYear, + currentMonth, + currentDate, + onClick, + elementRef, + activeDate, + ghost, + } = props; + + const handleClick = React.useCallback(() => { + if (onClick) { + onClick(year, month, date); + } + }, [year, month, date, onClick]); + + const dateString = ymdToDateString(year, month, date); + + return ( + + {date} + + + ); +} + +export default typedMemo(CalendarDate); diff --git a/manager-dashboard/app/components/Calendar/CalendarDate/styles.css b/manager-dashboard/app/components/Calendar/CalendarDate/styles.css new file mode 100644 index 000000000..cf87157ab --- /dev/null +++ b/manager-dashboard/app/components/Calendar/CalendarDate/styles.css @@ -0,0 +1,25 @@ +.date { + border-radius: 50%; + width: var(--width-calendar-date); + height: var(--width-calendar-date); + + &.today { + color: var(--color-accent); + font-weight: var(--font-weight-bold); + } + + &:hover { + background-color: var(--color-background-hover-light); + } + + &.active { + background-color: var(--color-accent); + color: var(--color-text-on-dark); + pointer-events: none; + } + + &.ghost { + opacity: 0.5; + } +} + diff --git a/manager-dashboard/app/components/Calendar/index.tsx b/manager-dashboard/app/components/Calendar/index.tsx new file mode 100644 index 000000000..d72f9054d --- /dev/null +++ b/manager-dashboard/app/components/Calendar/index.tsx @@ -0,0 +1,292 @@ +import React from 'react'; +import { + _cs, + isNotDefined, + isDefined, +} from '@togglecorp/fujs'; +import { + IoTimeOutline, + IoChevronForward, + IoChevronBack, + IoCalendarOutline, +} from 'react-icons/io5'; + +import Button from '../Button'; +import NumberInput from '../NumberInput'; +import SelectInput from '../SelectInput'; +import useInputState from '../../hooks/useInputState'; +import { typedMemo } from '../../utils/common.tsx'; + +import CalendarDate, { Props as CalendarDateProps } from './CalendarDate'; + +import styles from './styles.css'; + +const weekDayNames = [ + 'Sunday', + 'Monday', + 'Tuesday', + 'Wednesday', + 'Thursday', + 'Friday', + 'Saturday', +]; + +interface MonthName { + key: number; + label: string; +} + +const monthNameList: MonthName[] = [ + { key: 0, label: 'January' }, + { key: 1, label: 'February' }, + { key: 2, label: 'March' }, + { key: 3, label: 'April' }, + { key: 4, label: 'May' }, + { key: 5, label: 'June' }, + { key: 6, label: 'July' }, + { key: 7, label: 'August' }, + { key: 8, label: 'September' }, + { key: 9, label: 'October' }, + { key: 10, label: 'November' }, + { key: 11, label: 'December' }, +]; + +function getStartOfWeek(year: number, month: number) { + return new Date(year, month, 1).getDay(); +} + +function getNumDaysInMonth(year: number, month: number) { + // Setting date to 0 will switch the date to last day of previous month + return new Date(year, month + 1, 0).getDate(); +} + +interface RenderDate { + type: 'prevMonth' | 'currentMonth' | 'nextMonth'; + date: number; +} + +function getDates(year: number, month: number) { + const numDays = getNumDaysInMonth(year, month); + const numDayInPrevMonth = getNumDaysInMonth(year, month - 1); + const startOfWeek = getStartOfWeek(year, month); + + const dates: RenderDate[] = []; + + for (let i = 0; i < startOfWeek; i += 1) { + dates.push({ + type: 'prevMonth', + date: numDayInPrevMonth - startOfWeek + i + 1, + }); + } + + for (let i = 0; i < numDays; i += 1) { + dates.push({ + type: 'currentMonth', + date: i + 1, + }); + } + + // 6 rows x 7 cols + const remainingDates = 42 - dates.length; + + for (let i = 0; i < remainingDates; i += 1) { + dates.push({ + type: 'nextMonth', + date: i + 1, + }); + } + + return dates; +} + +const monthKeySelector = (m: MonthName) => m.key; +const monthLabelSelector = (m: MonthName) => m.label; + +type RendererOmissions = 'year' | 'month' | 'date' | 'currentYear' | 'currentMonth' | 'currentDate' | 'onClick' | 'activeDate' | 'ghost'; +export interface Props

{ + className?: string; + dateRenderer?: (props: P) => React.ReactElement; + rendererParams?: (day: number, month: number, year: number) => Omit; + onDateClick?: (day: number, month: number, year: number) => void; + monthSelectionPopupClassName?: string; + initialDate?: string; + activeDate?: string; +} + +function Calendar

(props: Props

) { + const { + className, + dateRenderer: DateRenderer = CalendarDate, + rendererParams, + onDateClick, + monthSelectionPopupClassName, + initialDate, + activeDate, + } = props; + + const today = new Date(); + const current = initialDate ? new Date(initialDate) : today; + const currentYear = current.getFullYear(); + const currentMonth = current.getMonth(); + + const [year, setYear] = useInputState(currentYear); + const [month, setMonth] = useInputState(currentMonth); + + const dates = year ? getDates(year, month) : undefined; + + const handleGotoCurrentButtonClick = React.useCallback(() => { + const date = new Date(); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + }, [setMonth, setYear]); + + const handleNextMonthButtonClick = React.useCallback(() => { + if (isDefined(year)) { + const date = new Date(year, month + 1, 1); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + } + }, [year, month, setMonth, setYear]); + + const handlePreviousMonthButtonClick = React.useCallback(() => { + if (isDefined(year)) { + const date = new Date(year, month - 1, 1); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + } + }, [year, month, setMonth, setYear]); + + const isValidYear = React.useMemo(() => { + if (isNotDefined(year)) { + return false; + } + + if (year < 1900 || year > 9999) { + return false; + } + + return true; + }, [year]); + + return ( +

+
+
+
+ +
+
+ +
+
+
+ {weekDayNames.map((wd) => ( +
+ {wd.substr(0, 2)} +
+ ))} +
+
+ {(isValidYear && isDefined(year) && dates) ? ( +
+ {dates.map((date) => { + let newMonth = month; + if (date.type === 'prevMonth') { + newMonth -= 1; + } else if (date.type === 'nextMonth') { + newMonth += 1; + } + const ymd = new Date(year, newMonth, date.date); + + const defaultProps: Pick = { + onClick: onDateClick, + year: ymd.getFullYear(), + month: ymd.getMonth(), + date: ymd.getDate(), + currentYear: today.getFullYear(), + currentMonth: today.getMonth(), + currentDate: today.getDate(), + activeDate, + ghost: date.type === 'prevMonth' || date.type === 'nextMonth', + }; + + const combinedProps = { + ...(rendererParams ? rendererParams( + date.date, month, year, + ) : undefined), + ...defaultProps, + } as P; + + const children = ( + + ); + + return ( +
+ {children} +
+ ); + })} +
+ ) : ( +
+ + Please select a valid year and month to view the dates +
+ )} +
+ + + +
+
+ ); +} + +export default typedMemo(Calendar); diff --git a/manager-dashboard/app/components/Calendar/styles.css b/manager-dashboard/app/components/Calendar/styles.css new file mode 100644 index 000000000..8d31cba85 --- /dev/null +++ b/manager-dashboard/app/components/Calendar/styles.css @@ -0,0 +1,93 @@ +.calendar { + display: flex; + flex-direction: column; + + .header { + flex-shrink: 0; + + .info { + display: flex; + align-items: flex-end; + justify-content: center; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .current-year { + flex-basis: 40%; + padding: var(--spacing-small); + font-size: var(--font-size-large); + } + + .current-month { + flex-basis: 60%; + padding: var(--spacing-small); + } + } + + .week-days { + display: flex; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .week-day-name { + display: flex; + align-items: center; + flex-basis: calc(100% / 7); + flex-shrink: 0; + justify-content: center; + padding: var(--spacing-small); + font-weight: var(--font-weight-bold); + } + } + } + + .day-list { + display: flex; + flex-grow: 1; + flex-wrap: wrap; + padding: calc(var(--spacing-medium) - var(--spacing-small)) var(--spacing-medium); + + .day-container { + --width: calc(100% / 7); + display: flex; + align-items: center; + flex-basis: var(--width); + justify-content: center; + width: var(--width); + } + } + + .empty-day-list { + display: flex; + align-items: center; + flex-direction: column; + flex-grow: 1; + justify-content: center; + padding: var(--spacing-large); + text-align: center; + color: var(--color-text); + + .icon { + opacity: var(--opacity-watermark); + margin: var(--spacing-medium); + font-size: var(--font-size-ultra-large); + } + } + + .actions { + display: flex; + flex-shrink: 0; + justify-content: flex-end; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + >* { + margin: var(--spacing-small) calc(var(--spacing-medium) - var(--spacing-small)); + } + } +} + +.month-selection-popup { + min-width: 10rem; + + .popup-content { + width: 100%!important; + } +} diff --git a/manager-dashboard/app/components/DateRangeInput/index.tsx b/manager-dashboard/app/components/DateRangeInput/index.tsx new file mode 100644 index 000000000..f7e12f53d --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/index.tsx @@ -0,0 +1,420 @@ +import React, { useMemo } from 'react'; +import { + _cs, + randomString, + isDefined, + isNotDefined, +} from '@togglecorp/fujs'; +import { + IoCalendarOutline, + IoClose, +} from 'react-icons/io5'; + +import useBlurEffect from '../../hooks/useBlurEffect'; +import useBooleanState from '../../hooks/useBooleanState'; +import InputContainer, { Props as InputContainerProps } from '../InputContainer'; +import RawInput from '../RawInput'; +import RawButton from '../RawButton'; +import Button from '../Button'; +import Popup from '../Popup'; +import Calendar, { Props as CalendarProps } from '../Calendar'; +import CalendarDate, { Props as CalendarDateProps } from '../Calendar/CalendarDate'; +import { ymdToDateString, dateStringToDate } from '../../utils/common.tsx'; + +import { + predefinedDateRangeOptions, + PredefinedDateRangeKey, +} from './predefinedDateRange'; + +import styles from './styles.css'; + +// FIXME: this is problematic when on end months +function prevMonth(date: Date) { + const newDate = new Date(date); + newDate.setMonth(newDate.getMonth() - 1); + return newDate; +} +function sameMonth(foo: Date, bar: Date) { + return foo.getFullYear() === bar.getFullYear() && foo.getMonth() === bar.getMonth(); +} + +export interface Value { + startDate: string; + endDate: string; +} + +interface DateRendererProps extends CalendarDateProps { + startDate?: string; + endDate?: string; +} + +function DateRenderer(props: DateRendererProps) { + const { + className: dateClassName, + year, + month, + date, + startDate, + endDate, + ghost, + ...otherProps + } = props; + + const start = startDate ? dateStringToDate(startDate).getTime() : undefined; + const end = endDate ? dateStringToDate(endDate).getTime() : undefined; + const current = new Date(year, month, date).getTime(); + + const inBetween = isDefined(start) && isDefined(end) && current > start && current < end; + + const dateString = ymdToDateString(year, month, date); + + const isEndDate = dateString === endDate; + const isStartDate = dateString === startDate; + + return ( + + ); +} + +type NameType = string | number | undefined; + +type InheritedProps = Omit; +export interface Props extends InheritedProps { + inputElementRef?: React.RefObject; + inputClassName?: string; + value: Value | undefined | null; + name: N; + onChange?: (value: Value | undefined, name: N) => void; + placeholder?: string; +} + +function DateRangeInput(props: Props) { + const { + actions, + actionsContainerClassName, + className, + disabled, + error, + errorContainerClassName, + hint, + hintContainerClassName, + icons, + iconsContainerClassName, + inputSectionClassName, + label, + labelContainerClassName, + readOnly, + inputElementRef, + containerRef: containerRefFromProps, + inputSectionRef: inputSectionRefFromProps, + inputClassName, + onChange, + name, + value, + placeholder, + } = props; + + const [tempDate, setTempDate] = React.useState>({ + startDate: undefined, + endDate: undefined, + }); + const [calendarMonthSelectionPopupClassName] = React.useState(randomString(16)); + const createdContainerRef = React.useRef(null); + const createdInputSectionRef = React.useRef(null); + const popupRef = React.useRef(null); + + const containerRef = containerRefFromProps ?? createdContainerRef; + const inputSectionRef = inputSectionRefFromProps ?? createdInputSectionRef; + const [ + showCalendar, + setShowCalendarTrue, + setShowCalendarFalse,, + toggleShowCalendar, + ] = useBooleanState(false); + + const hideCalendar = React.useCallback(() => { + setTempDate({ + startDate: undefined, + endDate: undefined, + }); + setShowCalendarFalse(); + }, [setShowCalendarFalse]); + + const handlePopupBlur = React.useCallback( + (isClickedWithin: boolean, e: MouseEvent) => { + // Following is to prevent the popup blur when + // month selection is changed in the calendar + const container = document.getElementsByClassName( + calendarMonthSelectionPopupClassName, + )[0]; + const isContainerOrInsideContainer = container + ? container === e.target || container.contains(e.target as HTMLElement) + : false; + if (!isClickedWithin && !isContainerOrInsideContainer) { + hideCalendar(); + } + }, + [hideCalendar, calendarMonthSelectionPopupClassName], + ); + + useBlurEffect( + showCalendar, + handlePopupBlur, + popupRef, + inputSectionRef, + ); + + const dateRendererParams = React.useCallback(() => ({ + startDate: tempDate.startDate ?? value?.startDate, + // we only set end date if user hasn't set the start date + // i.e. to show previously selected end date) + endDate: !tempDate.startDate ? value?.endDate : undefined, + }), [tempDate.startDate, value]); + + const handleCalendarDateClick: CalendarProps['onDateClick'] = React.useCallback( + (year, month, day) => { + setTempDate((prevTempDate) => { + if (isDefined(prevTempDate.startDate)) { + const lastDate = ymdToDateString(year, month, day); + + const prev = dateStringToDate(prevTempDate.startDate).getTime(); + const current = new Date(year, month, day).getTime(); + + const startDate = prev > current ? lastDate : prevTempDate.startDate; + const endDate = prev > current ? prevTempDate.startDate : lastDate; + + return { + startDate, + endDate, + }; + } + + return { + startDate: ymdToDateString(year, month, day), + endDate: undefined, + }; + }); + }, + [], + ); + + React.useEffect(() => { + if (isDefined(tempDate.endDate)) { + if (onChange) { + onChange(tempDate as Value, name); + } + hideCalendar(); + } + }, [tempDate, hideCalendar, onChange, name]); + + const handlePredefinedOptionClick = React.useCallback((optionKey: PredefinedDateRangeKey) => { + if (onChange) { + const option = predefinedDateRangeOptions.find((d) => d.key === optionKey); + + if (option) { + const { + startDate, + endDate, + } = option.getValue(); + + onChange({ + startDate: ymdToDateString( + startDate.getFullYear(), + startDate.getMonth(), + startDate.getDate(), + ), + endDate: ymdToDateString( + endDate.getFullYear(), + endDate.getMonth(), + endDate.getDate(), + ), + }, name); + } + } + + hideCalendar(); + }, [onChange, hideCalendar, name]); + + const handleClearButtonClick = React.useCallback(() => { + if (onChange) { + onChange(undefined, name); + } + }, [onChange, name]); + + const endDate = value?.endDate; + const endDateDate = endDate + ? dateStringToDate(endDate) + : new Date(); + + const startDate = value?.startDate; + let startDateDate = startDate + ? dateStringToDate(startDate) + : new Date(); + + if (sameMonth(endDateDate, startDateDate)) { + startDateDate = prevMonth(startDateDate); + } + + const firstInitialDate = ymdToDateString( + startDateDate.getFullYear(), + startDateDate.getMonth(), + 1, + ); + const secondInitialDate = ymdToDateString( + endDateDate.getFullYear(), + endDateDate.getMonth(), + 1, + ); + + const dateInputLabel = useMemo( + () => { + if ( + isNotDefined(tempDate.startDate) + && isNotDefined(value?.startDate) + && isNotDefined(value?.endDate) + ) { + return undefined; + } + + const startDateString = tempDate.startDate ?? value?.startDate; + const start = isDefined(startDateString) + ? new Date(startDateString).toLocaleDateString() + : '--'; + const endDateString = value?.endDate; + const end = isDefined(endDateString) + ? new Date(endDateString).toLocaleDateString() + : '--'; + + return [ + start, + end, + ].join(' to '); + }, + [value, tempDate], + ); + + return ( + <> + + { actions } + {!readOnly && ( + <> + {value && ( + + )} + + + )} + + )} + actionsContainerClassName={actionsContainerClassName} + className={className} + disabled={disabled} + error={error} + errorContainerClassName={errorContainerClassName} + hint={hint} + hintContainerClassName={hintContainerClassName} + icons={icons} + iconsContainerClassName={iconsContainerClassName} + inputSectionClassName={inputSectionClassName} + inputContainerClassName={styles.inputContainer} + label={label} + labelContainerClassName={labelContainerClassName} + readOnly={readOnly} + input={( + + )} + /> + {!readOnly && showCalendar && ( + +
+ {predefinedDateRangeOptions.map((opt) => ( + + {opt.label} + + ))} +
+ + + + + )} + + ); +} + +export default DateRangeInput; diff --git a/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts b/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts new file mode 100644 index 000000000..4764069c6 --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts @@ -0,0 +1,181 @@ +export type PredefinedDateRangeKey = 'today' + | 'yesterday' + | 'thisWeek' + | 'lastSevenDays' + | 'thisMonth' + | 'lastThirtyDays' + | 'lastThreeMonths' + | 'lastSixMonths' + | 'thisYear' + | 'lastYear'; + +export interface PredefinedDateRangeOption { + key: PredefinedDateRangeKey; + label: string; + getValue: () => ({ startDate: Date, endDate: Date }); +} + +export const predefinedDateRangeOptions: PredefinedDateRangeOption[] = [ + { + key: 'today', + label: 'Today', + getValue: () => ({ + startDate: new Date(), + endDate: new Date(), + }), + }, + { + key: 'yesterday', + label: 'Yesterday', + getValue: () => { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - 1); + + const endDate = new Date(); + endDate.setDate(endDate.getDate() - 1); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisWeek', + label: 'This week', + getValue: () => { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - startDate.getDay()); + + const endDate = new Date(); + // NOTE: this will give us sunday + endDate.setDate(startDate.getDate() + 6); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastSevenDays', + label: 'Last 7 days', + getValue: () => { + const endDate = new Date(); + + const startDate = new Date(); + startDate.setDate(endDate.getDate() - 7); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisMonth', + label: 'This month', + getValue: () => { + const startDate = new Date(); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastThirtyDays', + label: 'Last 30 days', + getValue: () => { + const endDate = new Date(); + + const startDate = new Date(); + startDate.setDate(endDate.getDate() - 30); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastThreeMonths', + label: 'Last 3 months', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(startDate.getMonth() - 2); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastSixMonths', + label: 'Last 6 months', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(startDate.getMonth() - 5); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisYear', + label: 'This year', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(0); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setFullYear(startDate.getFullYear() + 1); + endDate.setMonth(0); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastYear', + label: 'Last year', + getValue: () => { + const startDate = new Date(); + startDate.setFullYear(startDate.getFullYear() - 1); + startDate.setMonth(0); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(0); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, +]; diff --git a/manager-dashboard/app/components/DateRangeInput/styles.css b/manager-dashboard/app/components/DateRangeInput/styles.css new file mode 100644 index 000000000..4d4388b75 --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/styles.css @@ -0,0 +1,85 @@ +.input-container { + display: flex; + flex-direction: row; + + .input { + --color: var(--color-text); + flex-grow: 1; + min-width: unset; + color: var(--color); + + &.empty { + --color: var(--color-input-placeholder); /* TODO */ + } + + &.errored { + --color: var(--color-danger); + } + } +} + +.calendar-popup { + height: 25rem; + + .popup-content { + display: flex; + max-width: unset!important; + max-height: unset!important; + + .calendar { + --padding: var(--spacing-medium); + width: calc(var(--width-calendar-date) * 7 + 2 * var(--padding)); + height: 100%; + } + + .predefined-options { + display: flex; + flex-direction: column; + justify-content: center; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .option { + padding: var(--spacing-small); + width: 100%; + text-align: right; + + &:hover { + background-color: var(--color-background-hover-light); + } + } + } + } +} + +.calendar-date { + &.start-date { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + + &:not(.ghost) { + background-color: var(--color-accent); + color: var(--color-text-on-dark); + } + &.ghost { + background-color: var(--color-background-hover-light); + } + } + + &.end-date { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + + &:not(.ghost) { + background-color: var(--color-accent); + color: var(--color-text-ligth); + } + &.ghost { + background-color: var(--color-background-hover-light); + } + } + + &.in-between { + border-radius: 0; + background-color: var(--color-background-hover-light); + } +} diff --git a/manager-dashboard/app/utils/common.tsx b/manager-dashboard/app/utils/common.tsx index 1a37b3e8c..53338d34f 100644 --- a/manager-dashboard/app/utils/common.tsx +++ b/manager-dashboard/app/utils/common.tsx @@ -323,3 +323,17 @@ export const formatProjectTopic = (projectTopic: string) => { return newProjectTopic; }; + +export function ymdToDateString(year: number, month: number, day: number) { + const ys = String(year).padStart(4, '0'); + const ms = String(month + 1).padStart(2, '0'); + const ds = String(day).padStart(2, '0'); + + return `${ys}-${ms}-${ds}`; +} + +export function dateStringToDate(value: string) { + return new Date(`${value}T00:00`); +} + +export const typedMemo: ((c: T) => T) = React.memo; diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 7bad14c40..6c3c6bef1 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -52,6 +52,7 @@ import AnimatedSwipeIcon from '#components/AnimatedSwipeIcon'; import ExpandableContainer from '#components/ExpandableContainer'; import AlertBanner from '#components/AlertBanner'; import Checkbox from '#components/Checkbox'; +import DateRangeInput from '#components/DateRangeInput'; import { valueSelector, labelSelector, @@ -317,6 +318,9 @@ function NewProject(props: Props) { valuesToCopy.geometry = res.geometry; } + valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate || null; + valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate || null; + const storage = getStorage(); const timestamp = (new Date()).getTime(); const uploadedImageRef = storageRef(storage, `projectImages/${timestamp}-project-image-${projectImage.name}`); @@ -711,17 +715,18 @@ function NewProject(props: Props) { )} {value.projectType === PROJECT_TYPE_STREET && ( - /* TODO: Add street project inputs for - startTimestamp, - endTimeStamp, - isPano, - organizationId, - samplingThreshold - */ -
time range input
+ Date: Mon, 16 Dec 2024 16:01:30 +0100 Subject: [PATCH 065/107] fix: use clip of image locations to aoi --- .../utils/process_mapillary.py | 4 +--- .../tests/unittests/test_process_mapillary.py | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 4b6943fdf..44357755a 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -134,17 +134,15 @@ def coordinate_download( for col in target_columns: if col not in downloaded_metadata.columns: downloaded_metadata[col] = None - if ( downloaded_metadata.isna().all().all() is False - or downloaded_metadata.empty is True + or downloaded_metadata.empty is False ): downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply( lambda point: point.within(polygon) ) ] - return downloaded_metadata diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 8134ad2bf..1913ccb47 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -5,7 +5,7 @@ import pandas as pd from shapely import wkt -from shapely.geometry import GeometryCollection, MultiPolygon, Polygon +from shapely.geometry import GeometryCollection, MultiPolygon, Point, Polygon from mapswipe_workers.utils.process_mapillary import ( coordinate_download, @@ -191,10 +191,30 @@ def test_download_and_process_tile_failure(self, mock_get): @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") def test_coordinate_download(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = pd.DataFrame([{"geometry": None}]) + inside_points = [ + (0.2, 0.2), + (0.5, 0.5), + ] + outside_points = [ + (1.5, 0.5), + (0.5, 1.5), + (-0.5, 0.5), + ] + points = inside_points + outside_points + data = [ + { + "geometry": Point(x, y), + } + for x, y in points + ] + + mock_download_and_process_tile.return_value = pd.DataFrame(data) metadata = coordinate_download(self.test_polygon, self.level) + metadata = metadata.drop_duplicates() + self.assertEqual(len(metadata), len(inside_points)) + self.assertIsInstance(metadata, pd.DataFrame) @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") From fb6610efff780bbaa23d27858662ffbe804093c6 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 17 Dec 2024 12:21:54 +0100 Subject: [PATCH 066/107] feat: add filter by creator_id --- .../mapswipe_workers/project_types/street/project.py | 1 + .../mapswipe_workers/utils/process_mapillary.py | 12 +++++++++++- .../tests/unittests/test_process_mapillary.py | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index e15f42c1f..cc122c409 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -46,6 +46,7 @@ def __init__(self, project_draft): # TODO: validate inputs ImageMetadata = get_image_metadata( self.geometry, + creator_id=project_draft.get("creatorId", None), is_pano=project_draft.get("isPano", None), start_time=project_draft.get("startTimestamp", None), end_time=project_draft.get("endTimestamp", None), diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 44357755a..03f536706 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -184,12 +184,21 @@ def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None) def filter_results( results_df: pd.DataFrame, + creator_id: int = None, is_pano: bool = None, organization_id: str = None, start_time: str = None, end_time: str = None, ): df = results_df.copy() + if creator_id is not None: + if df["creator_id"].isna().all(): + logger.exception( + "No Mapillary Feature in the AoI has a 'creator_id' value." + ) + return None + df = df[df["creator_id"] == creator_id] + if is_pano is not None: if df["is_pano"].isna().all(): logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") @@ -220,6 +229,7 @@ def get_image_metadata( level=14, attempt_limit=3, is_pano: bool = None, + creator_id: int = None, organization_id: str = None, start_time: str = None, end_time: str = None, @@ -234,7 +244,7 @@ def get_image_metadata( ] downloaded_metadata = filter_results( - downloaded_metadata, is_pano, organization_id, start_time, end_time + downloaded_metadata, creator_id, is_pano, organization_id, start_time, end_time ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 1913ccb47..32c1bad46 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -263,6 +263,10 @@ def test_filter_organization_id(self): filtered_df = filter_results(self.fixture_df, organization_id=1) self.assertEqual(len(filtered_df), 1) + def test_filter_creator_id(self): + filtered_df = filter_results(self.fixture_df, creator_id=102506575322825) + self.assertEqual(len(filtered_df), 3) + def test_filter_time_range(self): start_time = "2016-01-20 00:00:00" end_time = "2022-01-21 23:59:59" From 88de7c2f003bb591dfd2caff041c5ee3dbd8c346 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Tue, 17 Dec 2024 13:50:22 +0100 Subject: [PATCH 067/107] feat(manager-dashboard): add creator id input to mapillary image filters --- manager-dashboard/app/views/NewProject/index.tsx | 15 ++++++++++++--- manager-dashboard/app/views/NewProject/utils.ts | 8 ++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 6c3c6bef1..9d4acf352 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -318,8 +318,8 @@ function NewProject(props: Props) { valuesToCopy.geometry = res.geometry; } - valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate || null; - valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate || null; + valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate ?? null; + valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate ?? null; const storage = getStorage(); const timestamp = (new Date()).getTime(); @@ -720,13 +720,22 @@ function NewProject(props: Props) { > + Date: Tue, 17 Dec 2024 14:13:16 +0100 Subject: [PATCH 068/107] feat(manager-dashboard): adjust value interface --- manager-dashboard/app/components/DateRangeInput/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manager-dashboard/app/components/DateRangeInput/index.tsx b/manager-dashboard/app/components/DateRangeInput/index.tsx index f7e12f53d..6442fc835 100644 --- a/manager-dashboard/app/components/DateRangeInput/index.tsx +++ b/manager-dashboard/app/components/DateRangeInput/index.tsx @@ -39,8 +39,8 @@ function sameMonth(foo: Date, bar: Date) { } export interface Value { - startDate: string; - endDate: string; + startDate?: string; + endDate?: string; } interface DateRendererProps extends CalendarDateProps { From 0f5406264515bb7d739c4e1b3224afd2cae670fb Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 17 Dec 2024 14:14:59 +0100 Subject: [PATCH 069/107] fix: exception handling when no elements are matched with filter --- .../utils/process_mapillary.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 03f536706..1faf0b23b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -237,31 +237,35 @@ def get_image_metadata( ): aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) - if downloaded_metadata.isna().all().all() or downloaded_metadata.empty: + + if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): raise ValueError("No Mapillary Features in the AoI.") + downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] - downloaded_metadata = filter_results( + filtered_metadata = filter_results( downloaded_metadata, creator_id, is_pano, organization_id, start_time, end_time ) - if sampling_threshold is not None: - downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) + if ( - downloaded_metadata.isna().all().all() is False - or downloaded_metadata.empty is False + filtered_metadata is None + or filtered_metadata.empty + or filtered_metadata.isna().all().all() ): - if len(downloaded_metadata) > 100000: - err = ( - f"Too many Images with selected filter " - f"options for the AoI: {len(downloaded_metadata)}" - ) - raise ValueError(err) - else: - return { - "ids": downloaded_metadata["id"].tolist(), - "geometries": downloaded_metadata["geometry"].tolist(), - } - else: raise ValueError("No Mapillary Features in the AoI match the filter criteria.") + + if sampling_threshold is not None: + filtered_metadata = spatial_sampling(filtered_metadata, sampling_threshold) + + total_images = len(filtered_metadata) + if total_images > 100000: + raise ValueError( + f"Too many Images with selected filter options for the AoI: {total_images}" + ) + + return { + "ids": filtered_metadata["id"].tolist(), + "geometries": filtered_metadata["geometry"].tolist(), + } From 4811d9426e59fa853798b0136d6904b441200483 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 16 Jan 2025 16:20:38 +0100 Subject: [PATCH 070/107] fix(manager-dashboard): correctly set isPano for street projects --- manager-dashboard/app/views/NewProject/index.tsx | 7 ++++--- manager-dashboard/app/views/NewProject/utils.ts | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 9d4acf352..8c378597c 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -108,7 +108,7 @@ const defaultProjectFormValue: PartialProjectFormType = { // maxTasksPerUser: -1, inputType: PROJECT_INPUT_TYPE_UPLOAD, filter: FILTER_BUILDINGS, - isPano: false, + panoOnly: false, }; interface Props { @@ -320,6 +320,7 @@ function NewProject(props: Props) { valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate ?? null; valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate ?? null; + valuesToCopy.isPano = valuesToCopy.panoOnly ? true : null; const storage = getStorage(); const timestamp = (new Date()).getTime(); @@ -756,8 +757,8 @@ function NewProject(props: Props) { disabled={submissionPending || projectTypeEmpty} /> Date: Mon, 20 Jan 2025 11:44:08 +0100 Subject: [PATCH 071/107] feat: sequences are now in direction of travel --- mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 3cafb5362..a35271fe3 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -146,4 +146,7 @@ def spatial_sampling(df, interval_length): [sampled_sequence_df, filtered_sorted_sub_df], axis=0 ) + # reverse order such that sequence are in direction of travel + sampled_sequence_df = sampled_sequence_df.iloc[::-1] + return sampled_sequence_df From 417a5fae9cd2a1599017851ad8bdb8e5bf8448f2 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 20 Jan 2025 11:47:59 +0100 Subject: [PATCH 072/107] fix: just use on dataframe to save memory --- .../mapswipe_workers/utils/process_mapillary.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 1faf0b23b..35dd96e30 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -245,27 +245,27 @@ def get_image_metadata( downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] - filtered_metadata = filter_results( + downloaded_metadata = filter_results( downloaded_metadata, creator_id, is_pano, organization_id, start_time, end_time ) if ( - filtered_metadata is None - or filtered_metadata.empty - or filtered_metadata.isna().all().all() + downloaded_metadata is None + or downloaded_metadata.empty + or downloaded_metadata.isna().all().all() ): raise ValueError("No Mapillary Features in the AoI match the filter criteria.") if sampling_threshold is not None: - filtered_metadata = spatial_sampling(filtered_metadata, sampling_threshold) + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - total_images = len(filtered_metadata) + total_images = len(downloaded_metadata) if total_images > 100000: raise ValueError( f"Too many Images with selected filter options for the AoI: {total_images}" ) return { - "ids": filtered_metadata["id"].tolist(), - "geometries": filtered_metadata["geometry"].tolist(), + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), } From 3f09ae7fff3821e1422a9016edad0d623a0b0e82 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 20 Jan 2025 14:59:50 +0100 Subject: [PATCH 073/107] feat: add option to randomize order of images --- .../project_types/street/project.py | 1 + .../mapswipe_workers/utils/process_mapillary.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py index cc122c409..4d0fe8c5b 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/project.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -51,6 +51,7 @@ def __init__(self, project_draft): start_time=project_draft.get("startTimestamp", None), end_time=project_draft.get("endTimestamp", None), organization_id=project_draft.get("organizationId", None), + randomize_order=project_draft.get("randomizeOrder", None), sampling_threshold=project_draft.get("samplingThreshold", None), ) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 35dd96e30..1c9159059 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -189,6 +189,7 @@ def filter_results( organization_id: str = None, start_time: str = None, end_time: str = None, + randomize_order: bool = None, ): df = results_df.copy() if creator_id is not None: @@ -221,6 +222,9 @@ def filter_results( return None df = filter_by_timerange(df, start_time, end_time) + if randomize_order is not None: + df.sample(frac=1).reset_index(drop=True) + return df @@ -233,6 +237,7 @@ def get_image_metadata( organization_id: str = None, start_time: str = None, end_time: str = None, + randomize_order=False, sampling_threshold=None, ): aoi_polygon = geojson_to_polygon(aoi_geojson) @@ -246,7 +251,13 @@ def get_image_metadata( ] downloaded_metadata = filter_results( - downloaded_metadata, creator_id, is_pano, organization_id, start_time, end_time + downloaded_metadata, + creator_id, + is_pano, + organization_id, + start_time, + end_time, + randomize_order, ) if ( From b9e0a495ef7a0e49764ff629bd6b1038b012edc2 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 20 Jan 2025 15:37:00 +0100 Subject: [PATCH 074/107] fix: change test to match new code --- .../tests/unittests/test_spatial_sampling.py | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py index f43597c89..8c5aea418 100644 --- a/mapswipe_workers/tests/unittests/test_spatial_sampling.py +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -1,29 +1,32 @@ import os - import unittest + import numpy as np import pandas as pd from shapely import wkt from shapely.geometry import Point -from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, spatial_sampling +from mapswipe_workers.utils.spatial_sampling import ( + distance_on_sphere, + filter_points, + spatial_sampling, +) class TestDistanceCalculations(unittest.TestCase): - @classmethod def setUpClass(cls): with open( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "fixtures", - "mapillary_sequence.csv", - ), - "r", + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_sequence.csv", + ), + "r", ) as file: df = pd.read_csv(file) - df['geometry'] = df['geometry'].apply(wkt.loads) + df["geometry"] = df["geometry"].apply(wkt.loads) cls.fixture_df = df @@ -42,41 +45,43 @@ def test_filter_points(self): "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", - "POINT (-77.006 43.7128)" + "POINT (-77.006 43.7128)", ] } df = pd.DataFrame(data) - df['geometry'] = df['geometry'].apply(wkt.loads) + df["geometry"] = df["geometry"].apply(wkt.loads) - df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None) - df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None) + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) threshold_distance = 100 filtered_df = filter_points(df, threshold_distance) self.assertIsInstance(filtered_df, pd.DataFrame) self.assertLessEqual(len(filtered_df), len(df)) - def test_spatial_sampling_ordering(self): data = { "geometry": [ "POINT (-74.006 40.7128)", "POINT (-75.006 41.7128)", "POINT (-76.006 42.7128)", - "POINT (-77.006 43.7128)" + "POINT (-77.006 43.7128)", ], - 'captured_at': [1, 2, 3, 4], - 'sequence_id': ['1', '1', '1', '1'] + "captured_at": [1, 2, 3, 4], + "sequence_id": ["1", "1", "1", "1"], } df = pd.DataFrame(data) - df['geometry'] = df['geometry'].apply(wkt.loads) + df["geometry"] = df["geometry"].apply(wkt.loads) interval_length = 0.1 filtered_gdf = spatial_sampling(df, interval_length) - self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing) - + self.assertTrue(filtered_gdf["captured_at"].is_monotonic_decreasing) def test_spatial_sampling_with_sequence(self): threshold_distance = 0.01 @@ -86,15 +91,13 @@ def test_spatial_sampling_with_sequence(self): filtered_df.reset_index(drop=True, inplace=True) for i in range(len(filtered_df) - 1): - geom1 = filtered_df.loc[i, 'geometry'] - geom2 = filtered_df.loc[i + 1, 'geometry'] + geom1 = filtered_df.loc[i, "geometry"] + geom2 = filtered_df.loc[i + 1, "geometry"] distance = geom1.distance(geom2) self.assertLess(distance, threshold_distance) - - if __name__ == "__main__": unittest.main() From b8b2c14065ce8e449f5619f9ad2794d19935235d Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 22 Jan 2025 18:10:52 +0100 Subject: [PATCH 075/107] feat(manager-dashboard): Add randomizeOrder input for new Street projects --- manager-dashboard/app/views/NewProject/index.tsx | 8 ++++++++ manager-dashboard/app/views/NewProject/utils.ts | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index 9d4acf352..2f88a42ab 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -109,6 +109,7 @@ const defaultProjectFormValue: PartialProjectFormType = { inputType: PROJECT_INPUT_TYPE_UPLOAD, filter: FILTER_BUILDINGS, isPano: false, + randomizeOrder: false, }; interface Props { @@ -762,6 +763,13 @@ function NewProject(props: Props) { onChange={setFieldValue} disabled={submissionPending || projectTypeEmpty} /> +
)} diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index 57efed00d..e9ea77e8d 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -84,6 +84,7 @@ export interface ProjectFormType { organizationId?: number; creatorId?: number; isPano?: boolean; + randomizeOrder?: boolean; samplingThreshold?: number; } @@ -308,6 +309,9 @@ export const projectFormSchema: ProjectFormSchema = { isPano: { required: false, }, + randomizeOrder: { + required: false, + }, }; baseSchema = addCondition( From e2486d9b61d8679000b1fa462706a20e3b1cd5e9 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 27 Jan 2025 11:39:19 +0100 Subject: [PATCH 076/107] fix: remove artifacts from debugging --- mapswipe_workers/tests/unittests/test_tutorial.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_tutorial.py b/mapswipe_workers/tests/unittests/test_tutorial.py index 51a4cbf84..5ba1c209a 100644 --- a/mapswipe_workers/tests/unittests/test_tutorial.py +++ b/mapswipe_workers/tests/unittests/test_tutorial.py @@ -1,27 +1,26 @@ import os import unittest -from mapswipe_workers.project_types import StreetTutorial +from mapswipe_workers.project_types import ClassificationTutorial from tests.fixtures import FIXTURE_DIR, get_fixture class TestTutorial(unittest.TestCase): def test_init_tile_classification_project(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") ) - self.assertIsNotNone(StreetTutorial(tutorial_draft=tutorial_draft)) + self.assertIsNotNone(ClassificationTutorial(tutorial_draft=tutorial_draft)) def test_create_tile_classification_tasks(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") ) - tutorial = StreetTutorial(tutorial_draft=tutorial_draft) + tutorial = ClassificationTutorial(tutorial_draft=tutorial_draft) tutorial.create_tutorial_groups() tutorial.create_tutorial_tasks() self.assertTrue(tutorial.groups) self.assertTrue(tutorial.tasks) - breakpoint() if __name__ == "__main__": From bc2bcd5c3a7b549c23911337a8431e2535e7030b Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 27 Jan 2025 15:48:51 +0100 Subject: [PATCH 077/107] feat: use ProcessPoolExecutor for download_and_process_tile --- .../utils/process_mapillary.py | 144 +++++++----------- 1 file changed, 59 insertions(+), 85 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 1faf0b23b..ffec6de8d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -1,18 +1,11 @@ import os -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ProcessPoolExecutor +from functools import partial import mercantile import pandas as pd import requests -from shapely import ( - LineString, - MultiLineString, - MultiPolygon, - Point, - Polygon, - box, - unary_union, -) +from shapely import MultiPolygon, Point, Polygon, box, unary_union from shapely.geometry import shape from vt2geojson import tools as vt2geojson_tools @@ -44,7 +37,7 @@ def create_tiles(polygon, level): return tiles -def download_and_process_tile(row, attempt_limit=3): +def download_and_process_tile(row, polygon, kwargs, attempt_limit=3): z = row["z"] x = row["x"] y = row["y"] @@ -59,31 +52,37 @@ def download_and_process_tile(row, attempt_limit=3): "features", [] ) data = [] - for feature in features: - geometry = feature.get("geometry", {}) - properties = feature.get("properties", {}) - geometry_type = geometry.get("type", None) - coordinates = geometry.get("coordinates", []) - - element_geometry = None - if geometry_type == "Point": - element_geometry = Point(coordinates) - elif geometry_type == "LineString": - element_geometry = LineString(coordinates) - elif geometry_type == "MultiLineString": - element_geometry = MultiLineString(coordinates) - elif geometry_type == "Polygon": - element_geometry = Polygon(coordinates[0]) - elif geometry_type == "MultiPolygon": - element_geometry = MultiPolygon(coordinates) - - # Append the dictionary with geometry and properties - row = {"geometry": element_geometry, **properties} - data.append(row) + data.extend( + [ + { + "geometry": Point(feature["geometry"]["coordinates"]), + **feature.get("properties", {}), + } + for feature in features + if feature.get("geometry", {}).get("type") == "Point" + ] + ) data = pd.DataFrame(data) - if not data.empty: + if data.isna().all().all() is False or data.empty is False: + data = data[data["geometry"].apply(lambda point: point.within(polygon))] + target_columns = [ + "id", + "geometry", + "captured_at", + "is_pano", + "compass_angle", + "sequence", + "organization_id", + ] + for col in target_columns: + if col not in data.columns: + data[col] = None + + if data.isna().all().all() is False or data.empty is False: + data = filter_results(data, **kwargs) + return data except Exception as e: print(f"An exception occurred while requesting a tile: {e}") @@ -94,7 +93,7 @@ def download_and_process_tile(row, attempt_limit=3): def coordinate_download( - polygon, level, use_concurrency=True, attempt_limit=3, workers=os.cpu_count() * 4 + polygon, level, kwargs: dict, use_concurrency=True, workers=os.cpu_count() * 4 ): tiles = create_tiles(polygon, level) @@ -104,45 +103,22 @@ def coordinate_download( if not use_concurrency: workers = 1 - futures = [] - with ThreadPoolExecutor(max_workers=workers) as executor: - for index, row in tiles.iterrows(): - futures.append( - executor.submit(download_and_process_tile, row, attempt_limit) - ) - - for future in as_completed(futures): - if future is not None: - df = future.result() + process_tile_with_args = partial( + download_and_process_tile, polygon=polygon, kwargs=kwargs + ) + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = list( + executor.map(process_tile_with_args, tiles.to_dict(orient="records")) + ) - if df is not None and not df.empty: - downloaded_metadata.append(df) + for df in futures: + if df is not None and not df.empty: + downloaded_metadata.append(df) if len(downloaded_metadata): downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: return pd.DataFrame(downloaded_metadata) - target_columns = [ - "id", - "geometry", - "captured_at", - "is_pano", - "compass_angle", - "sequence", - "organization_id", - ] - for col in target_columns: - if col not in downloaded_metadata.columns: - downloaded_metadata[col] = None - if ( - downloaded_metadata.isna().all().all() is False - or downloaded_metadata.empty is False - ): - downloaded_metadata = downloaded_metadata[ - downloaded_metadata["geometry"].apply( - lambda point: point.within(polygon) - ) - ] return downloaded_metadata @@ -198,13 +174,12 @@ def filter_results( ) return None df = df[df["creator_id"] == creator_id] - if is_pano is not None: if df["is_pano"].isna().all(): + print(df) logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] - if organization_id is not None: if df["organization_id"].isna().all(): logger.exception( @@ -212,7 +187,6 @@ def filter_results( ) return None df = df[df["organization_id"] == organization_id] - if start_time is not None: if df["captured_at"].isna().all(): logger.exception( @@ -220,14 +194,12 @@ def filter_results( ) return None df = filter_by_timerange(df, start_time, end_time) - return df def get_image_metadata( aoi_geojson, level=14, - attempt_limit=3, is_pano: bool = None, creator_id: int = None, organization_id: str = None, @@ -235,9 +207,15 @@ def get_image_metadata( end_time: str = None, sampling_threshold=None, ): + kwargs = { + "is_pano": is_pano, + "creator_id": creator_id, + "organization_id": organization_id, + "start_time": start_time, + "end_time": end_time, + } aoi_polygon = geojson_to_polygon(aoi_geojson) - downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) - + downloaded_metadata = coordinate_download(aoi_polygon, level, kwargs) if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): raise ValueError("No Mapillary Features in the AoI.") @@ -245,27 +223,23 @@ def get_image_metadata( downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) ] - filtered_metadata = filter_results( - downloaded_metadata, creator_id, is_pano, organization_id, start_time, end_time - ) - if ( - filtered_metadata is None - or filtered_metadata.empty - or filtered_metadata.isna().all().all() + downloaded_metadata is None + or downloaded_metadata.empty + or downloaded_metadata.isna().all().all() ): raise ValueError("No Mapillary Features in the AoI match the filter criteria.") if sampling_threshold is not None: - filtered_metadata = spatial_sampling(filtered_metadata, sampling_threshold) + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) - total_images = len(filtered_metadata) + total_images = len(downloaded_metadata) if total_images > 100000: raise ValueError( f"Too many Images with selected filter options for the AoI: {total_images}" ) return { - "ids": filtered_metadata["id"].tolist(), - "geometries": filtered_metadata["geometry"].tolist(), + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), } From c3dca086d7a76b41c62a8f7893d4b6d5ad0666c3 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 27 Jan 2025 15:53:35 +0100 Subject: [PATCH 078/107] fix(street): get_image_metadata() error handling --- .../mapswipe_workers/utils/process_mapillary.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 1faf0b23b..25213faf6 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -16,7 +16,12 @@ from shapely.geometry import shape from vt2geojson import tools as vt2geojson_tools -from mapswipe_workers.definitions import MAPILLARY_API_KEY, MAPILLARY_API_LINK, logger +from mapswipe_workers.definitions import ( + MAPILLARY_API_KEY, + MAPILLARY_API_LINK, + CustomError, + logger, +) from mapswipe_workers.utils.spatial_sampling import spatial_sampling @@ -239,7 +244,7 @@ def get_image_metadata( downloaded_metadata = coordinate_download(aoi_polygon, level, attempt_limit) if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): - raise ValueError("No Mapillary Features in the AoI.") + raise CustomError("No Mapillary Features in the AoI.") downloaded_metadata = downloaded_metadata[ downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) @@ -254,14 +259,14 @@ def get_image_metadata( or filtered_metadata.empty or filtered_metadata.isna().all().all() ): - raise ValueError("No Mapillary Features in the AoI match the filter criteria.") + raise CustomError("No Mapillary Features in the AoI match the filter criteria.") if sampling_threshold is not None: filtered_metadata = spatial_sampling(filtered_metadata, sampling_threshold) total_images = len(filtered_metadata) if total_images > 100000: - raise ValueError( + raise CustomError( f"Too many Images with selected filter options for the AoI: {total_images}" ) From cb12624d050546fd8a438d958ad255a629d1e14e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 27 Jan 2025 15:56:01 +0100 Subject: [PATCH 079/107] fix: tasks are only randomized if randomize_order is True --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 1c9159059..17d592c77 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -222,7 +222,7 @@ def filter_results( return None df = filter_by_timerange(df, start_time, end_time) - if randomize_order is not None: + if randomize_order is True: df.sample(frac=1).reset_index(drop=True) return df From 6e785980d71656171d983249dd969a5cd7a43fef Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 27 Jan 2025 16:27:16 +0100 Subject: [PATCH 080/107] fix: remove articat from debugging --- mapswipe_workers/tests/integration/test_create_tutorial.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py index 5ae13c76d..b61eb182e 100644 --- a/mapswipe_workers/tests/integration/test_create_tutorial.py +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -10,8 +10,8 @@ class TestCreateTileClassificationProject(unittest.TestCase): def setUp(self): self.tutorial_id = set_up.create_test_tutorial_draft( - "street", - "street", + "tile_classification", + "tile_classification", "test_tile_classification_tutorial", ) @@ -70,8 +70,6 @@ def test_create_tile_classification_project(self): result = ref.get(shallow=True) self.assertEqual(self.tutorial_id, result) - breakpoint() - if __name__ == "__main__": unittest.main() From 538c763ba882b8cfc14c4efc4660472129823215 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Mon, 27 Jan 2025 16:33:55 +0100 Subject: [PATCH 081/107] fix(street): adjust tests --- mapswipe_workers/tests/unittests/test_process_mapillary.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 32c1bad46..573a259f9 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -7,6 +7,7 @@ from shapely import wkt from shapely.geometry import GeometryCollection, MultiPolygon, Point, Polygon +from mapswipe_workers.definitions import CustomError from mapswipe_workers.utils.process_mapillary import ( coordinate_download, create_tiles, @@ -326,7 +327,7 @@ def test_get_image_metadata_no_rows(self, mock_coordinate_download): "start_time": "1916-01-20 00:00:00", "end_time": "1922-01-21 23:59:59", } - with self.assertRaises(ValueError): + with self.assertRaises(CustomError): get_image_metadata(self.fixture_data, **params) @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") @@ -335,7 +336,7 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): df = df.drop(df.index) mock_coordinate_download.return_value = df - with self.assertRaises(ValueError): + with self.assertRaises(CustomError): get_image_metadata(self.fixture_data) @patch("mapswipe_workers.utils.process_mapillary.filter_results") @@ -346,7 +347,7 @@ def test_get_image_metadata_size_restriction( mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) mock_coordinate_download.return_value = self.fixture_df - with self.assertRaises(ValueError): + with self.assertRaises(CustomError): get_image_metadata(self.fixture_data) From d4dd6a0655dec987e98877c39a5099683ba8b8e2 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 13:36:17 +0100 Subject: [PATCH 082/107] refactor: extract functions for improved testing --- .../utils/process_mapillary.py | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index ffec6de8d..a8f7f62f6 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -46,25 +46,7 @@ def download_and_process_tile(row, polygon, kwargs, attempt_limit=3): attempt = 0 while attempt < attempt_limit: try: - r = requests.get(url) - assert r.status_code == 200, r.content - features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( - "features", [] - ) - data = [] - data.extend( - [ - { - "geometry": Point(feature["geometry"]["coordinates"]), - **feature.get("properties", {}), - } - for feature in features - if feature.get("geometry", {}).get("type") == "Point" - ] - ) - - data = pd.DataFrame(data) - + data = get_mapillary_data(url, x, y, z) if data.isna().all().all() is False or data.empty is False: data = data[data["geometry"].apply(lambda point: point.within(polygon))] target_columns = [ @@ -79,7 +61,6 @@ def download_and_process_tile(row, polygon, kwargs, attempt_limit=3): for col in target_columns: if col not in data.columns: data[col] = None - if data.isna().all().all() is False or data.empty is False: data = filter_results(data, **kwargs) @@ -92,6 +73,26 @@ def download_and_process_tile(row, polygon, kwargs, attempt_limit=3): return None +def get_mapillary_data(url, x, y, z): + r = requests.get(url) + assert r.status_code == 200, r.content + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( + "features", [] + ) + data = [] + data.extend( + [ + { + "geometry": Point(feature["geometry"]["coordinates"]), + **feature.get("properties", {}), + } + for feature in features + if feature.get("geometry", {}).get("type") == "Point" + ] + ) + return pd.DataFrame(data) + + def coordinate_download( polygon, level, kwargs: dict, use_concurrency=True, workers=os.cpu_count() * 4 ): @@ -103,18 +104,11 @@ def coordinate_download( if not use_concurrency: workers = 1 - process_tile_with_args = partial( - download_and_process_tile, polygon=polygon, kwargs=kwargs + downloaded_metadata = parallelized_processing( + downloaded_metadata, kwargs, polygon, tiles, workers ) - with ProcessPoolExecutor(max_workers=workers) as executor: - futures = list( - executor.map(process_tile_with_args, tiles.to_dict(orient="records")) - ) - - for df in futures: - if df is not None and not df.empty: - downloaded_metadata.append(df) if len(downloaded_metadata): + breakpoint() downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: return pd.DataFrame(downloaded_metadata) @@ -122,6 +116,21 @@ def coordinate_download( return downloaded_metadata +def parallelized_processing(data, kwargs, polygon, tiles, workers): + process_tile_with_args = partial( + download_and_process_tile, polygon=polygon, kwargs=kwargs + ) + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = list( + executor.map(process_tile_with_args, tiles.to_dict(orient="records")) + ) + + for df in futures: + if df is not None and not df.empty: + data.append(df) + return data + + def geojson_to_polygon(geojson_data): if geojson_data["type"] == "FeatureCollection": features = geojson_data["features"] @@ -176,7 +185,6 @@ def filter_results( df = df[df["creator_id"] == creator_id] if is_pano is not None: if df["is_pano"].isna().all(): - print(df) logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] @@ -219,10 +227,6 @@ def get_image_metadata( if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): raise ValueError("No Mapillary Features in the AoI.") - downloaded_metadata = downloaded_metadata[ - downloaded_metadata["geometry"].apply(lambda geom: isinstance(geom, Point)) - ] - if ( downloaded_metadata is None or downloaded_metadata.empty From 87d65cb895a3551dfa7cd66807d8c561f62bfca3 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 13:37:47 +0100 Subject: [PATCH 083/107] fix: adapt tests to new multiprocessing --- .../tests/unittests/test_process_mapillary.py | 62 +++++-------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 32c1bad46..9fd1b4d1c 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -53,6 +53,7 @@ def setUp(self): ) self.empty_polygon = Polygon() self.empty_geometry = GeometryCollection() + self.row = pd.Series({"x": 1, "y": 1, "z": self.level}) def test_create_tiles_with_valid_polygon(self): tiles = create_tiles(self.test_polygon, self.level) @@ -171,26 +172,26 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): row = {"x": 1, "y": 1, "z": 14} - result = download_and_process_tile(row) + polygon = wkt.loads("POLYGON ((-1 -1, -1 1, 1 1, 1 -1, -1 -1))") + result = download_and_process_tile(row, polygon, {}) self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 1) self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") @patch("mapswipe_workers.utils.process_mapillary.requests.get") def test_download_and_process_tile_failure(self, mock_get): - # Mock a failed response + mock_response = MagicMock() mock_response.status_code = 500 mock_get.return_value = mock_response - row = pd.Series({"x": 1, "y": 1, "z": self.level}) - result = download_and_process_tile(row) + result = download_and_process_tile(self.row, self.test_polygon, {}) self.assertIsNone(result) - @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") - def test_coordinate_download(self, mock_download_and_process_tile): + @patch("mapswipe_workers.utils.process_mapillary.get_mapillary_data") + def test_download_and_process_tile_spatial_filtering(self, mock_get_mapillary_data): inside_points = [ (0.2, 0.2), (0.5, 0.5), @@ -208,20 +209,20 @@ def test_coordinate_download(self, mock_download_and_process_tile): for x, y in points ] - mock_download_and_process_tile.return_value = pd.DataFrame(data) + mock_get_mapillary_data.return_value = pd.DataFrame(data) - metadata = coordinate_download(self.test_polygon, self.level) + metadata = download_and_process_tile(self.row, self.test_polygon, {}) metadata = metadata.drop_duplicates() self.assertEqual(len(metadata), len(inside_points)) self.assertIsInstance(metadata, pd.DataFrame) - @patch("mapswipe_workers.utils.process_mapillary.download_and_process_tile") - def test_coordinate_download_with_failures(self, mock_download_and_process_tile): - mock_download_and_process_tile.return_value = pd.DataFrame() + @patch("mapswipe_workers.utils.process_mapillary.parallelized_processing") + def test_coordinate_download_with_failures(self, mock_parallelized_processing): + mock_parallelized_processing.return_value = pd.DataFrame() - metadata = coordinate_download(self.test_polygon, self.level) + metadata = coordinate_download(self.test_polygon, self.level, {}) self.assertTrue(metadata.empty) @@ -284,7 +285,7 @@ def test_filter_missing_columns(self): "is_pano", "organization_id", "captured_at", - ] # Add your column names here + ] for column in columns_to_check: df_copy = self.fixture_df.copy() df_copy[column] = None @@ -302,33 +303,6 @@ def test_get_image_metadata(self, mock_coordinate_download): self.assertIn("ids", result) self.assertIn("geometries", result) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") - def test_get_image_metadata_filtering(self, mock_coordinate_download): - mock_coordinate_download.return_value = self.fixture_df - - params = { - "is_pano": True, - "start_time": "2016-01-20 00:00:00", - "end_time": "2022-01-21 23:59:59", - } - - result = get_image_metadata(self.fixture_data, **params) - self.assertIsInstance(result, dict) - self.assertIn("ids", result) - self.assertIn("geometries", result) - - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") - def test_get_image_metadata_no_rows(self, mock_coordinate_download): - mock_coordinate_download.return_value = self.fixture_df - - params = { - "is_pano": True, - "start_time": "1916-01-20 00:00:00", - "end_time": "1922-01-21 23:59:59", - } - with self.assertRaises(ValueError): - get_image_metadata(self.fixture_data, **params) - @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_empty_response(self, mock_coordinate_download): df = self.fixture_df.copy() @@ -338,13 +312,9 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) - @patch("mapswipe_workers.utils.process_mapillary.filter_results") @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") - def test_get_image_metadata_size_restriction( - self, mock_coordinate_download, mock_filter_results - ): - mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) - mock_coordinate_download.return_value = self.fixture_df + def test_get_image_metadata_size_restriction(self, mock_coordinate_download): + mock_coordinate_download.return_value = pd.DataFrame({"ID": range(1, 100002)}) with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) From 41d641363d7eecf78acc6d8b8e111e7ea5fe2111 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 13:58:59 +0100 Subject: [PATCH 084/107] fix: remove debugging artifact --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index a8f7f62f6..b2dd3c76c 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -108,7 +108,6 @@ def coordinate_download( downloaded_metadata, kwargs, polygon, tiles, workers ) if len(downloaded_metadata): - breakpoint() downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) else: return pd.DataFrame(downloaded_metadata) From 23b654e569e0a89ea552738cd6d29192841b2d4f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 17:22:42 +0100 Subject: [PATCH 085/107] fix: remove debugging artifact --- .../tests/integration/test_create_street_project.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py index 10da2bd0e..fd0608f98 100644 --- a/mapswipe_workers/tests/integration/test_create_street_project.py +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -56,8 +56,6 @@ def test_create_street_project(self): result = ref.get(shallow=True) self.assertIsNotNone(result) - breakpoint() - if __name__ == "__main__": unittest.main() From 7899bfbb4200ebdbea357944511b3a10cd28519e Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 18:02:42 +0100 Subject: [PATCH 086/107] refactor: use logger.info instead of logger.exception for missing values --- .../mapswipe_workers/utils/process_mapillary.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 25213faf6..6c866e43d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -198,21 +198,19 @@ def filter_results( df = results_df.copy() if creator_id is not None: if df["creator_id"].isna().all(): - logger.exception( - "No Mapillary Feature in the AoI has a 'creator_id' value." - ) + logger.info("No Mapillary Feature in the AoI has a 'creator_id' value.") return None df = df[df["creator_id"] == creator_id] if is_pano is not None: if df["is_pano"].isna().all(): - logger.exception("No Mapillary Feature in the AoI has a 'is_pano' value.") + logger.info("No Mapillary Feature in the AoI has a 'is_pano' value.") return None df = df[df["is_pano"] == is_pano] if organization_id is not None: if df["organization_id"].isna().all(): - logger.exception( + logger.info( "No Mapillary Feature in the AoI has an 'organization_id' value." ) return None @@ -220,9 +218,7 @@ def filter_results( if start_time is not None: if df["captured_at"].isna().all(): - logger.exception( - "No Mapillary Feature in the AoI has a 'captured_at' value." - ) + logger.info("No Mapillary Feature in the AoI has a 'captured_at' value.") return None df = filter_by_timerange(df, start_time, end_time) From 7ff268877a88eb2804861e40dd2129733a9a9969 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 28 Jan 2025 18:17:25 +0100 Subject: [PATCH 087/107] fix: requesting the same tile multiple times for multipolygon --- .../mapswipe_workers/utils/process_mapillary.py | 4 ++-- .../tests/unittests/test_process_mapillary.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 1faf0b23b..aaad7a508 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -26,9 +26,9 @@ def create_tiles(polygon, level): if isinstance(polygon, Polygon): polygon = MultiPolygon([polygon]) - tiles = [] + tiles = set() for i, poly in enumerate(polygon.geoms): - tiles.extend(list(mercantile.tiles(*poly.bounds, level))) + tiles.update(list(mercantile.tiles(*poly.bounds, level))) bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] bbox_polygons = [box(*bbox) for bbox in bbox_list] diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 32c1bad46..6438ac96c 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -48,9 +48,6 @@ def setUpClass(cls): def setUp(self): self.level = 14 self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) - self.test_multipolygon = MultiPolygon( - [self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])] - ) self.empty_polygon = Polygon() self.empty_geometry = GeometryCollection() @@ -60,9 +57,19 @@ def test_create_tiles_with_valid_polygon(self): self.assertFalse(tiles.empty) def test_create_tiles_with_multipolygon(self): - tiles = create_tiles(self.test_multipolygon, self.level) + polygon = Polygon( + [ + (0.00000000, 0.00000000), + (0.000000001, 0.00000000), + (0.00000000, 0.000000001), + (0.00000000, 0.000000001), + ] + ) + multipolygon = MultiPolygon([polygon, polygon]) + tiles = create_tiles(multipolygon, self.level) self.assertIsInstance(tiles, pd.DataFrame) self.assertFalse(tiles.empty) + self.assertEqual(len(tiles), 1) def test_create_tiles_with_empty_polygon(self): tiles = create_tiles(self.empty_polygon, self.level) From 4d7790bf02b5ee5a342767e09e4854a03ee8fd8f Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 30 Jan 2025 14:09:13 +0100 Subject: [PATCH 088/107] fix: remove unnecessary check for empty dataframe --- .../mapswipe_workers/utils/process_mapillary.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index b2dd3c76c..a87c1970b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -224,14 +224,9 @@ def get_image_metadata( aoi_polygon = geojson_to_polygon(aoi_geojson) downloaded_metadata = coordinate_download(aoi_polygon, level, kwargs) if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): - raise ValueError("No Mapillary Features in the AoI.") - - if ( - downloaded_metadata is None - or downloaded_metadata.empty - or downloaded_metadata.isna().all().all() - ): - raise ValueError("No Mapillary Features in the AoI match the filter criteria.") + raise ValueError( + "No Mapillary Features in the AoI or no Features match the filter criteria." + ) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) From 916d4f703b5a9c3731b004eb592f904b69d2cff4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 30 Jan 2025 14:21:49 +0100 Subject: [PATCH 089/107] feat: drop duplicated images at exact same location --- .../utils/process_mapillary.py | 2 +- .../tests/unittests/test_process_mapillary.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index a87c1970b..ac06f435e 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -227,7 +227,7 @@ def get_image_metadata( raise ValueError( "No Mapillary Features in the AoI or no Features match the filter criteria." ) - + downloaded_metadata = downloaded_metadata.drop_duplicates(subset=["geometry"]) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 9fd1b4d1c..e6894fae1 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -314,11 +314,28 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") def test_get_image_metadata_size_restriction(self, mock_coordinate_download): - mock_coordinate_download.return_value = pd.DataFrame({"ID": range(1, 100002)}) + mock_coordinate_download.return_value = pd.DataFrame( + {"geometry": range(1, 100002)} + ) with self.assertRaises(ValueError): get_image_metadata(self.fixture_data) + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_drop_duplicates(self, mock_coordinate_download): + test_df = pd.DataFrame( + { + "id": [1, 2, 2, 3, 4, 4, 5], + "geometry": ["a", "b", "b", "c", "d", "d", "e"], + } + ) + mock_coordinate_download.return_value = test_df + return_dict = get_image_metadata(self.fixture_data) + + return_df = pd.DataFrame(return_dict) + + self.assertNotEqual(len(return_df), len(test_df)) + if __name__ == "__main__": unittest.main() From 50cb94fe0680fdef8a61fe062fcf3806f5425d31 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 3 Feb 2025 11:33:01 +0100 Subject: [PATCH 090/107] fix: randomize after spatial sampling --- .../mapswipe_workers/utils/process_mapillary.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 17d592c77..7067115c2 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -189,7 +189,6 @@ def filter_results( organization_id: str = None, start_time: str = None, end_time: str = None, - randomize_order: bool = None, ): df = results_df.copy() if creator_id is not None: @@ -222,9 +221,6 @@ def filter_results( return None df = filter_by_timerange(df, start_time, end_time) - if randomize_order is True: - df.sample(frac=1).reset_index(drop=True) - return df @@ -257,7 +253,6 @@ def get_image_metadata( organization_id, start_time, end_time, - randomize_order, ) if ( @@ -270,6 +265,9 @@ def get_image_metadata( if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) + if randomize_order is True: + downloaded_metadata.sample(frac=1).reset_index(drop=True) + total_images = len(downloaded_metadata) if total_images > 100000: raise ValueError( From 8b372843f6ad646b926fcec16f3784c84f01667b Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 3 Feb 2025 17:49:10 +0100 Subject: [PATCH 091/107] fix: reassign downloaded metadata --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 7067115c2..30828d77b 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -266,7 +266,7 @@ def get_image_metadata( downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if randomize_order is True: - downloaded_metadata.sample(frac=1).reset_index(drop=True) + downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True) total_images = len(downloaded_metadata) if total_images > 100000: From 3bdc562852dd600700ee00f3b09ae51d3c1395e4 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Tue, 4 Feb 2025 17:04:17 +0100 Subject: [PATCH 092/107] feat: images are in sequential order if randomizeOrder is false or none --- .../mapswipe_workers/utils/process_mapillary.py | 3 +-- .../mapswipe_workers/utils/spatial_sampling.py | 7 +++---- mapswipe_workers/tests/unittests/test_process_mapillary.py | 6 +++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index 30828d77b..bc0cf737d 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -262,8 +262,7 @@ def get_image_metadata( ): raise ValueError("No Mapillary Features in the AoI match the filter criteria.") - if sampling_threshold is not None: - downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if randomize_order is True: downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True) diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index a35271fe3..97302b945 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -141,10 +141,9 @@ def spatial_sampling(df, interval_length): for sequence in sorted_df["sequence_id"].unique(): sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] - filtered_sorted_sub_df = filter_points(sequence_df, interval_length) - sampled_sequence_df = pd.concat( - [sampled_sequence_df, filtered_sorted_sub_df], axis=0 - ) + if interval_length: + sequence_df = filter_points(sequence_df, interval_length) + sampled_sequence_df = pd.concat([sampled_sequence_df, sequence_df], axis=0) # reverse order such that sequence are in direction of travel sampled_sequence_df = sampled_sequence_df.iloc[::-1] diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py index 32c1bad46..ec19cbf78 100644 --- a/mapswipe_workers/tests/unittests/test_process_mapillary.py +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -343,7 +343,11 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download): def test_get_image_metadata_size_restriction( self, mock_coordinate_download, mock_filter_results ): - mock_filter_results.return_value = pd.DataFrame({"ID": range(1, 100002)}) + mock_df = pd.DataFrame({"ID": range(1, 100002)}) + mock_df["geometry"] = self.test_polygon + mock_df["captured_at"] = range(1, 100002) + mock_df["sequence_id"] = 1 + mock_filter_results.return_value = mock_df mock_coordinate_download.return_value = self.fixture_df with self.assertRaises(ValueError): From dedd45506f38bfcf96f7c50ee314180778284fe8 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 6 Feb 2025 16:07:14 +0100 Subject: [PATCH 093/107] feat(manager-dashboard): show custom options input for street tutorials --- manager-dashboard/app/views/NewTutorial/index.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/manager-dashboard/app/views/NewTutorial/index.tsx b/manager-dashboard/app/views/NewTutorial/index.tsx index 76f238066..a6fc640ce 100644 --- a/manager-dashboard/app/views/NewTutorial/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/index.tsx @@ -69,6 +69,7 @@ import { PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_FOOTPRINT, + PROJECT_TYPE_STREET, ProjectType, projectTypeLabelMap, } from '#utils/common'; @@ -761,7 +762,10 @@ function NewTutorial(props: Props) { autoFocus /> - {value.projectType === PROJECT_TYPE_FOOTPRINT && ( + {( + value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_STREET + ) && ( Date: Thu, 6 Feb 2025 16:58:11 +0100 Subject: [PATCH 094/107] feat(manager-dashboard): handle default custom options on project type change --- .../app/views/NewTutorial/index.tsx | 4 +- .../app/views/NewTutorial/utils.ts | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/manager-dashboard/app/views/NewTutorial/index.tsx b/manager-dashboard/app/views/NewTutorial/index.tsx index a6fc640ce..9ce9d4b67 100644 --- a/manager-dashboard/app/views/NewTutorial/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/index.tsx @@ -77,7 +77,7 @@ import { import { tileServerUrls, tutorialFormSchema, - defaultFootprintCustomOptions, + getDefaultOptions, TutorialFormType, PartialTutorialFormType, PartialInformationPagesType, @@ -342,7 +342,6 @@ const defaultTutorialFormValue: PartialTutorialFormType = { name: TILE_SERVER_ESRI, credits: tileServerDefaultCredits[TILE_SERVER_ESRI], }, - customOptions: defaultFootprintCustomOptions, }; type SubmissionStatus = 'started' | 'imageUpload' | 'tutorialSubmit' | 'success' | 'failed'; @@ -717,6 +716,7 @@ function NewTutorial(props: Props) { setFieldValue(undefined, 'tutorialTasks'); setFieldValue(undefined, 'scenarioPages'); setFieldValue(newValue, 'projectType'); + setFieldValue(getDefaultOptions(newValue), 'customOptions'); }, [setFieldValue], ); diff --git a/manager-dashboard/app/views/NewTutorial/utils.ts b/manager-dashboard/app/views/NewTutorial/utils.ts index 90b805f5d..4fc6deebf 100644 --- a/manager-dashboard/app/views/NewTutorial/utils.ts +++ b/manager-dashboard/app/views/NewTutorial/utils.ts @@ -26,6 +26,7 @@ import { PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_FOOTPRINT, + PROJECT_TYPE_STREET, IconKey, } from '#utils/common'; @@ -257,6 +258,33 @@ export const defaultFootprintCustomOptions: PartialTutorialFormType['customOptio }, ]; +export const defaultStreetCustomOptions: PartialTutorialFormType['customOptions'] = [ + { + optionId: 1, + value: 1, + title: 'Yes', + icon: 'checkmark-outline', + iconColor: colorKeyToColorMap.green, + description: '', + }, + { + optionId: 2, + value: 0, + title: 'No', + icon: 'close-outline', + iconColor: colorKeyToColorMap.red, + description: '', + }, + { + optionId: 3, + value: 2, + title: 'Not Sure', + icon: 'remove-outline', + iconColor: colorKeyToColorMap.gray, + description: 'if you\'re not sure or there is bad imagery', + }, +]; + export function deleteKey( value: T, key: K, @@ -268,6 +296,18 @@ export function deleteKey( return copy; } +export function getDefaultOptions(projectType: ProjectType | undefined) { + if (projectType === PROJECT_TYPE_FOOTPRINT) { + return defaultFootprintCustomOptions; + } + + if (projectType === PROJECT_TYPE_STREET) { + return defaultStreetCustomOptions; + } + + return undefined; +} + export interface BuildAreaProperties { reference: number; screen: number; From 3826747f7ac486fe3aa64dfdd7dceb7042a3bf50 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 6 Feb 2025 17:06:50 +0100 Subject: [PATCH 095/107] feat(manager-dashboard): hide tileserver input for street tutorials --- .../app/views/NewTutorial/index.tsx | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/manager-dashboard/app/views/NewTutorial/index.tsx b/manager-dashboard/app/views/NewTutorial/index.tsx index 9ce9d4b67..dccf7f4d6 100644 --- a/manager-dashboard/app/views/NewTutorial/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/index.tsx @@ -646,6 +646,11 @@ function NewTutorial(props: Props) { || tutorialSubmissionStatus === 'tutorialSubmit' ); + const tileServerVisible = value.projectType === PROJECT_TYPE_BUILD_AREA + || value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_CHANGE_DETECTION; + const tileServerBVisible = value.projectType === PROJECT_TYPE_CHANGE_DETECTION || value.projectType === PROJECT_TYPE_COMPLETENESS; @@ -900,17 +905,20 @@ function NewTutorial(props: Props) { )} - - - + {tileServerVisible && ( + + + + )} + {tileServerBVisible && ( Date: Wed, 19 Feb 2025 16:28:15 +0100 Subject: [PATCH 096/107] feat(street-tutorials): add street tutorial sample scenario and instructions README --- mapswipe_workers/sample_data/street/README.md | 58 +++++++++++++++++++ .../street_tutorial_sample_scenario.geojson | 17 ++++++ 2 files changed, 75 insertions(+) create mode 100644 mapswipe_workers/sample_data/street/README.md create mode 100644 mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson diff --git a/mapswipe_workers/sample_data/street/README.md b/mapswipe_workers/sample_data/street/README.md new file mode 100644 index 000000000..dfa52293e --- /dev/null +++ b/mapswipe_workers/sample_data/street/README.md @@ -0,0 +1,58 @@ +# Creating a New 'Street' Tutorial +### Useful Links +- MapSwipe Development Server: [https://dev-managers.mapswipe.org] +- MapSwipe Development App Installation Guide: [https://github.com/mapswipe/mapswipe/wiki/How-to-test-the-development-version-of-MapSwipe](https://github.com/mapswipe/mapswipe/wiki/How-to-test-the-development-version-of-MapSwipe) + +## Select appropriate Mapillary imagery for the tutorial (with JOSM and Mapillary plug-in) + +1. Open JOSM. Make sure the [JOSM Mapillary plug-in](https://wiki.openstreetmap.org/wiki/JOSM/Plugins/Mapillary) is installed +2. **File > Download data**. Select an area in which you expect appropriate example imagery available on Mapillary and **Download** +3. **Imagery > Mapillary** to download sequences and images for the current area +4. If helpful, use the Mapillary filter dialog to filter images (for start and end date, user and/or organization) +5. Click **Mapillary** in Layers controls to select the Mapillary layer +6. Zoom in until you can see images location markers (green dots) +7. Click on the dots to view the images +8. Once you have found an image that you would like to use in your tutorial, **File > Export Mapillary images** and select **Export selected images** +9. Click **Explore** +10. Choose a parent folder for all images in this tutorial +11. **OK** +12. Repeat until you have exported all the images that you would like to use in the tutorial. Use the same parent folder for all images. + +## Add exported Mapillary images as geotagged images in QGIS + +1. Open QGIS +2. **Processing Toolbox > Vector creation > Import geotagged photos** +3. Select the folder containing all exported Mapillary images and check **Scan recursively** +4. **Run** +5. **Properties > Display** and add `` to HTML Map Tip to show images on a pop up +6. **View > Show Map Tips** +7. If you keep the mouse tip on the image markers, a pop up with the image will appear + +## Edit geotagged images in QGIS + +1. Right click on layer. +2. **Properties > Field** +3. **Toggle editing mode** +4. Change the name of the `filename` column to `id` +5. Add `Integer (32 bit)` columns titled `screen` and `reference`. +6. Populate the `reference` and `screen` fields. + * `reference` is the value of the correct answer option for the image. + * `screen` determines the order of the images in the tutorial and should start with `1`. +7. Delete any rows representing images that you do not want to use + +## Export as GeoJSON + +1. **Toggle editing mode** +2. **Save** +3. Right click, **Export > Save Features As...** +4. Choose Format GeoJSON, CRS EPSG:4326 - WGS 84 +5. Select only `id`, `reference` and `screen` as fields to export. Deselect all other fields. +6. Choose a file name and location and click OK to save + +## Create tutorial + +1. Go to https://dev-managers.mapswipe.org/ +2. Select **Projects** and then **Add New Tutorial**. +3. Check that **Project Type** is set to **Street**. +4. Fill in all the fields, following the instructions. Upload your `GeoJSON` you just created with the scenarios where it says **Scenario Pages**. +5. Submit diff --git a/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson new file mode 100644 index 000000000..3a56f6d50 --- /dev/null +++ b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson @@ -0,0 +1,17 @@ +{ + "type": "FeatureCollection", + "name": "cobblestone-scenario", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { "id": "378811598610667", "reference": 0, "screen": 2 }, + "geometry": { "type": "Point", "coordinates": [ 13.45285, 52.508467, 0.0 ] } + }, + { + "type": "Feature", + "properties": { "id": "1171343450849316", "reference": 1, "screen": 1 }, + "geometry": { "type": "Point", "coordinates": [ 13.4514123, 52.5103378, 0.0 ] } + } + ] +} From 32ce566d6c31d493de6b5f13dc5faa445ebe9360 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Wed, 19 Feb 2025 16:42:24 +0100 Subject: [PATCH 097/107] feat(street-tutorials): prepare scenario pages input for street tutorial --- .../views/NewTutorial/ScenarioPageInput/index.tsx | 10 +++++++++- manager-dashboard/app/views/NewTutorial/index.tsx | 5 +++++ manager-dashboard/app/views/NewTutorial/utils.ts | 13 ++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx index 607434590..b309be7ff 100644 --- a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx @@ -17,6 +17,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, + PROJECT_TYPE_STREET, } from '#utils/common'; import TextInput from '#components/TextInput'; import Heading from '#components/Heading'; @@ -318,7 +319,14 @@ export default function ScenarioPageInput(props: Props) { lookFor={lookFor} /> )} - {(projectType && projectType !== PROJECT_TYPE_FOOTPRINT) && ( + {projectType === PROJECT_TYPE_STREET && ( +
+ Preview not available. +
+ )} + {(projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_STREET) && ( checkSchema( diff --git a/manager-dashboard/app/views/NewTutorial/utils.ts b/manager-dashboard/app/views/NewTutorial/utils.ts index 4fc6deebf..be3a92fa0 100644 --- a/manager-dashboard/app/views/NewTutorial/utils.ts +++ b/manager-dashboard/app/views/NewTutorial/utils.ts @@ -348,6 +348,12 @@ export interface ChangeDetectionProperties { // taskId: string; } +export interface StreetProperties { + id: string; + reference: number; + screen: number; +} + export type BuildAreaGeoJSON = GeoJSON.FeatureCollection< GeoJSON.Geometry, BuildAreaProperties @@ -363,9 +369,14 @@ export type ChangeDetectionGeoJSON = GeoJSON.FeatureCollection< ChangeDetectionProperties >; +export type StreetGeoJSON = GeoJSON.FeatureCollection< + GeoJSON.Geometry, + StreetProperties +>; + export type TutorialTasksGeoJSON = GeoJSON.FeatureCollection< GeoJSON.Geometry, - BuildAreaProperties | FootprintProperties | ChangeDetectionProperties + BuildAreaProperties | FootprintProperties | ChangeDetectionProperties | StreetProperties >; export type CustomOptions = { From dfb374329cc71389ca49698804de96684db6270b Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 20 Feb 2025 12:18:38 +0100 Subject: [PATCH 098/107] feat: change structure of street tutorial to match those of other project types --- .../project_types/street/tutorial.py | 10 +- .../tests/fixtures/tutorialDrafts/street.json | 198 ++++++------------ 2 files changed, 74 insertions(+), 134 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py index ca2c56cbe..e59a97f09 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -24,7 +24,7 @@ def __init__(self, tutorial_draft): # self.projectId = tutorial_draft["projectId"] self.projectType = tutorial_draft["projectType"] - self.tutorial_tasks = tutorial_draft["tasks"] + self.tutorial_tasks = tutorial_draft["tutorialTasks"] self.groups = dict() self.tasks = dict() @@ -54,14 +54,14 @@ def create_tutorial_groups(self): def create_tutorial_tasks(self): """Create the tasks dict based on provided examples in geojson file.""" task_list = [] - for i, task in enumerate(self.tutorial_tasks): + for i, task in enumerate(self.tutorial_tasks["features"]): task = StreetTutorialTask( projectId=self.projectId, groupId=101, - taskId=f"{task['taskImageId']}", + taskId=f"{task['properties']['id']}", geometry="", - referenceAnswer=task["referenceAnswer"], - screen=i, + referenceAnswer=task["properties"]["reference"], + screen=task["properties"]["screen"], ) task_list.append(asdict(task)) if task_list: diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json index 22116363b..1385ffb52 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json @@ -1,170 +1,110 @@ { - "createdBy": "LtCUyou6CnSSc1H0Q0nDrN97x892", - "tutorialDraftId": "waste_mapping_dar_es_salaam", - "taskImageIds": [ - 888464808378923, - 1552821322020271, - 2969692853315413, - 1036040467918366, - 837497816845037 - ], - "answers": [ - 1, - 2, - 1, - 2, - 3 - ], - "customOptions": [ - { - "description": "the shape does outline a building in the image", - "icon": "hand-right-outline", - "iconColor": "#00796B", - "subOptions": [ - { - "description": "doppelt", - "value": 2 - }, - { - "description": "dreifach", - "value": 3 - } - ], - "title": "Jetzt rede ich", - "value": 1 - }, - { - "description": "the shape doesn't match a building in the image", - "icon": "close-outline", - "iconColor": "#D32F2F", - "title": "No", - "value": 0 - } - ], + "createdBy": "atCSosZACaN0qhcVjtMO1tq9d1G3", + "tutorialDraftId": "test_tile_classification", "informationPages": [ { "blocks": [ { "blockNumber": 1, "blockType": "text", - "textDescription": "asdf" + "textDescription": "This is the first information page" }, { "blockNumber": 2, "blockType": "image", - "image": "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/tutorialImages%2F1705402528654-block-image-2-base-query-form.png?alt=media&token=54325ab8-c5e7-45a3-be41-1926a5984a05" + "image": "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/tutorialImages%2F1739963139725-block-image-2-1x1.png?alt=media&token=ae584dcd-d351-4bfe-be5f-1e0d38547f72" } ], "pageNumber": 1, - "title": "asdf" + "title": "Information page 1" } ], - "lookFor": "waste", - "name": "Waste Mapping Dar es Salaam", + "lookFor": "cobblestone", + "name": "cobblestone-tutorial", "projectType": 7, "screens": [ null, { "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." + "description": "This seems to be a tarmac surface.", + "icon": "check", + "title": "Tarmac" }, "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" + "description": "Check out if the road surface material is cobblestone here", + "icon": "check", + "title": "Is this cobblestone?" }, "success": { - "description": "Swipe to the next screen to look for more.", + "description": "Correct, this is not cobblestone", "icon": "check", - "title": "You found your first areas with buildings!" + "title": "Nice!" } }, { "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." + "description": "That surface does look like cobblestone!", + "icon": "heart-outline", + "title": "Cobblestone" }, "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" + "description": "Does this look like cobblestone?", + "icon": "egg-outline", + "title": "How about this one?" }, "success": { - "description": "Swipe to the next screen to look for more.", - "icon": "check", - "title": "You found your first" - } + "description": "Correct", + "icon": "search-outline", + "title": "Correct" + } + } + ], + "tileServer": { + "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO", + "name": "bing" + }, + "tutorialTasks": { + "crs": { + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + }, + "type": "name" }, - { - "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." - }, - "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" - }, - "success": { - "description": "Swipe to the next screen to look for more.", - "icon": "check", - "title": "You found your first areas with buildings!" - } - }, - { - "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." + "features": [ + { + "geometry": { + "coordinates": [ + 13.4514123, + 52.5103378, + 0 + ], + "type": "Point" }, - "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" + "properties": { + "id": "1171343450849316", + "reference": 1, + "screen": 1 }, - "success": { - "description": "Swipe to the next screen to look for more.", - "icon": "check", - "title": "You found your first areas with buildings!" - } - }, - { - "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." + "type": "Feature" + }, + { + "geometry": { + "coordinates": [ + 13.45285, + 52.508467, + 0 + ], + "type": "Point" }, - "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" + "properties": { + "id": "378811598610667", + "reference": 0, + "screen": 2 }, - "success": { - "description": "Swipe to the next screen to look for more.", - "icon": "check", - "title": "You found your first areas with buildings!" - } - }, - { - "hint": { - "description": "Swipe to learn some more.", - "icon": "swipe-left", - "title": "We've marked the correct square green." - }, - "instructions": { - "description": "Tap once to mark the squares with buildings green.", - "icon": "tap-1", - "title": "There are some buildings in this image" - }, - "success": { - "description": "Swipe to the next screen to look for more.", - "icon": "check", - "title": "You found your first areas with buildings!" + "type": "Feature" } - } - ] + ], + "name": "cobblestone-scenario", + "type": "FeatureCollection" + } } \ No newline at end of file From 550df7a237036e9217e58d745f19293375da6453 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 20 Feb 2025 12:36:50 +0100 Subject: [PATCH 099/107] fix: working integration test for tutorial creation --- .../tests/integration/tear_down.py | 8 ++- .../tests/integration/test_create_tutorial.py | 50 ++++--------------- .../tests/unittests/test_tutorial.py | 10 ++-- 3 files changed, 22 insertions(+), 46 deletions(-) diff --git a/mapswipe_workers/tests/integration/tear_down.py b/mapswipe_workers/tests/integration/tear_down.py index 61760781c..33af5d52d 100644 --- a/mapswipe_workers/tests/integration/tear_down.py +++ b/mapswipe_workers/tests/integration/tear_down.py @@ -8,7 +8,7 @@ from mapswipe_workers import auth -def delete_test_data(project_id: str) -> None: +def delete_test_data(project_id: str, tutorial_id: str = None) -> None: """ Delete test project indluding groups, tasks and results from Firebase and Postgres @@ -38,6 +38,12 @@ def delete_test_data(project_id: str) -> None: ref = fb_db.reference(f"v2/users/{project_id}") ref.delete() + if tutorial_id is not None: + ref = fb_db.reference(f"v2/projects/{tutorial_id}") + ref.delete() + ref = fb_db.reference(f"v2/tutorialDrafts/{tutorial_id}") + ref.delete() + # Clear out the user-group used in test. # XXX: Use a firebase simulator for running test. # For CI/CD, use a real firebase with scope using commit hash, diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py index b61eb182e..31e45b3fa 100644 --- a/mapswipe_workers/tests/integration/test_create_tutorial.py +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -10,65 +10,35 @@ class TestCreateTileClassificationProject(unittest.TestCase): def setUp(self): self.tutorial_id = set_up.create_test_tutorial_draft( - "tile_classification", - "tile_classification", + "street", + "street", "test_tile_classification_tutorial", ) self.project_id = set_up.create_test_project_draft( - "tile_classification", - "tile_classification", + "street", + "street", "test_tile_classification_tutorial", tutorial_id=self.tutorial_id, ) create_directories() def tearDown(self): - tear_down.delete_test_data(self.project_id) + tear_down.delete_test_data(self.project_id, self.tutorial_id) def test_create_tile_classification_project(self): runner = CliRunner() + runner.invoke(mapswipe_workers.run_create_tutorials, catch_exceptions=False) runner.invoke(mapswipe_workers.run_create_projects, catch_exceptions=False) - pg_db = auth.postgresDB() - query = "SELECT project_id FROM projects WHERE project_id = %s" - result = pg_db.retr_query(query, [self.project_id])[0][0] - self.assertEqual(result, self.project_id) - - query = """ - SELECT project_id - FROM projects - WHERE project_id = %s - and project_type_specifics::jsonb ? 'customOptions' - """ - result = pg_db.retr_query(query, [self.project_id])[0][0] - self.assertEqual(result, self.project_id) - - query = "SELECT count(*) FROM groups WHERE project_id = %s" - result = pg_db.retr_query(query, [self.project_id])[0][0] - self.assertEqual(result, 20) - - query = "SELECT count(*) FROM tasks WHERE project_id = %s" - result = pg_db.retr_query(query, [self.project_id])[0][0] - self.assertEqual(result, 5040) - fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/projects/{self.project_id}") - result = ref.get(shallow=True) - self.assertIsNotNone(result) + result = ref.get() + self.assertEqual(result["tutorialId"], self.tutorial_id) - ref = fb_db.reference(f"/v2/groups/{self.project_id}") + ref = fb_db.reference(f"/v2/projects/{self.tutorial_id}") result = ref.get(shallow=True) - self.assertEqual(len(result), 20) - - # Tile classification projects do not have tasks in Firebase - ref = fb_db.reference(f"/v2/tasks/{self.project_id}") - result = ref.get(shallow=True) - self.assertIsNone(result) - - ref = fb_db.reference(f"/v2/projects/{self.project_id}/tutorialId") - result = ref.get(shallow=True) - self.assertEqual(self.tutorial_id, result) + self.assertIsNotNone(result) if __name__ == "__main__": diff --git a/mapswipe_workers/tests/unittests/test_tutorial.py b/mapswipe_workers/tests/unittests/test_tutorial.py index 5ba1c209a..16e9e6aa4 100644 --- a/mapswipe_workers/tests/unittests/test_tutorial.py +++ b/mapswipe_workers/tests/unittests/test_tutorial.py @@ -1,22 +1,22 @@ import os import unittest -from mapswipe_workers.project_types import ClassificationTutorial +from mapswipe_workers.project_types import StreetTutorial from tests.fixtures import FIXTURE_DIR, get_fixture class TestTutorial(unittest.TestCase): def test_init_tile_classification_project(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) - self.assertIsNotNone(ClassificationTutorial(tutorial_draft=tutorial_draft)) + self.assertIsNotNone(StreetTutorial(tutorial_draft=tutorial_draft)) def test_create_tile_classification_tasks(self): tutorial_draft = get_fixture( - os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) - tutorial = ClassificationTutorial(tutorial_draft=tutorial_draft) + tutorial = StreetTutorial(tutorial_draft=tutorial_draft) tutorial.create_tutorial_groups() tutorial.create_tutorial_tasks() self.assertTrue(tutorial.groups) From 5237ce07d4ac4e4981b8fe5a75d838e1603603f8 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 20 Feb 2025 12:38:00 +0100 Subject: [PATCH 100/107] fix: naming in integration test --- .../tests/integration/test_create_tutorial.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py index 31e45b3fa..e6db39579 100644 --- a/mapswipe_workers/tests/integration/test_create_tutorial.py +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -10,14 +10,14 @@ class TestCreateTileClassificationProject(unittest.TestCase): def setUp(self): self.tutorial_id = set_up.create_test_tutorial_draft( - "street", - "street", + "tile_classification", + "tile_classification", "test_tile_classification_tutorial", ) self.project_id = set_up.create_test_project_draft( - "street", - "street", + "tile_classification", + "tile_classification", "test_tile_classification_tutorial", tutorial_id=self.tutorial_id, ) @@ -26,7 +26,7 @@ def setUp(self): def tearDown(self): tear_down.delete_test_data(self.project_id, self.tutorial_id) - def test_create_tile_classification_project(self): + def test_create_tile_classification_project_and_tutorial(self): runner = CliRunner() runner.invoke(mapswipe_workers.run_create_tutorials, catch_exceptions=False) runner.invoke(mapswipe_workers.run_create_projects, catch_exceptions=False) From e2efb18a07c60e3aa28936863e30cc768d67fee8 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Thu, 20 Feb 2025 12:40:32 +0100 Subject: [PATCH 101/107] refactor: change name of test_tutorial to test_tutorial_street --- .../unittests/{test_tutorial.py => test_tutorial_street.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename mapswipe_workers/tests/unittests/{test_tutorial.py => test_tutorial_street.py} (87%) diff --git a/mapswipe_workers/tests/unittests/test_tutorial.py b/mapswipe_workers/tests/unittests/test_tutorial_street.py similarity index 87% rename from mapswipe_workers/tests/unittests/test_tutorial.py rename to mapswipe_workers/tests/unittests/test_tutorial_street.py index 16e9e6aa4..6dd9b0127 100644 --- a/mapswipe_workers/tests/unittests/test_tutorial.py +++ b/mapswipe_workers/tests/unittests/test_tutorial_street.py @@ -6,13 +6,13 @@ class TestTutorial(unittest.TestCase): - def test_init_tile_classification_project(self): + def test_init_street_tutorial(self): tutorial_draft = get_fixture( os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) self.assertIsNotNone(StreetTutorial(tutorial_draft=tutorial_draft)) - def test_create_tile_classification_tasks(self): + def test_create_street_tasks(self): tutorial_draft = get_fixture( os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") ) From f3ae5498d91b79c706227a0646137a836e876d98 Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 20 Feb 2025 14:01:02 +0100 Subject: [PATCH 102/107] feat(street-tutorial): correct reference value in sample data --- .../street/street_tutorial_sample_scenario.geojson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson index 3a56f6d50..8f5b236f0 100644 --- a/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson +++ b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson @@ -5,12 +5,12 @@ "features": [ { "type": "Feature", - "properties": { "id": "378811598610667", "reference": 0, "screen": 2 }, + "properties": { "id": "378811598610667", "reference": 1, "screen": 2 }, "geometry": { "type": "Point", "coordinates": [ 13.45285, 52.508467, 0.0 ] } }, { "type": "Feature", - "properties": { "id": "1171343450849316", "reference": 1, "screen": 1 }, + "properties": { "id": "1171343450849316", "reference": 0, "screen": 1 }, "geometry": { "type": "Point", "coordinates": [ 13.4514123, 52.5103378, 0.0 ] } } ] From c27e228588193c0112e30bdea036375ae12c393b Mon Sep 17 00:00:00 2001 From: Oliver Fritz Date: Thu, 20 Feb 2025 15:08:44 +0100 Subject: [PATCH 103/107] feat(street-tutorial): ensure that custom options are written to tutorial and project drafts --- manager-dashboard/app/views/NewProject/utils.ts | 3 ++- manager-dashboard/app/views/NewTutorial/utils.ts | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index c410084b0..685ffd39d 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -324,7 +324,8 @@ export const projectFormSchema: ProjectFormSchema = { ['projectType'], ['customOptions'], (formValues) => { - if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT) { + if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: { keySelector: (key) => key.value, diff --git a/manager-dashboard/app/views/NewTutorial/utils.ts b/manager-dashboard/app/views/NewTutorial/utils.ts index be3a92fa0..67f5e4af5 100644 --- a/manager-dashboard/app/views/NewTutorial/utils.ts +++ b/manager-dashboard/app/views/NewTutorial/utils.ts @@ -265,7 +265,7 @@ export const defaultStreetCustomOptions: PartialTutorialFormType['customOptions' title: 'Yes', icon: 'checkmark-outline', iconColor: colorKeyToColorMap.green, - description: '', + description: 'the object you are looking for is in the image.', }, { optionId: 2, @@ -273,7 +273,7 @@ export const defaultStreetCustomOptions: PartialTutorialFormType['customOptions' title: 'No', icon: 'close-outline', iconColor: colorKeyToColorMap.red, - description: '', + description: 'the object you are looking for is NOT in the image.', }, { optionId: 3, @@ -775,7 +775,8 @@ export const tutorialFormSchema: TutorialFormSchema = { }), }; - if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT) { + if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: customOptionField, }; From 64553efdad31066d0c7d53fbaa54fdec7027f449 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 24 Feb 2025 17:07:15 +0100 Subject: [PATCH 104/107] fix: remove duplicates after spatial sampling --- mapswipe_workers/mapswipe_workers/utils/process_mapillary.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py index f8db99cc9..54afc2be6 100644 --- a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -229,13 +229,14 @@ def get_image_metadata( raise CustomError( "No Mapillary Features in the AoI or no Features match the filter criteria." ) - downloaded_metadata = downloaded_metadata.drop_duplicates(subset=["geometry"]) if sampling_threshold is not None: downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) if randomize_order is True: downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True) + downloaded_metadata = downloaded_metadata.drop_duplicates(subset=["geometry"]) + total_images = len(downloaded_metadata) if total_images > 100000: raise CustomError( From 2e2152cc3b1b0536b4a3b58fa057da23c7b6b549 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 3 Mar 2025 15:04:51 +0100 Subject: [PATCH 105/107] feat: add unittest for each tutorial --- .../tutorialDrafts/change_detection.json | 1 + .../fixtures/tutorialDrafts/completeness.json | 1 + .../fixtures/tutorialDrafts/footprint.json | 1 + ...t_tutorial_arbitrary_geometry_footprint.py | 27 +++++++++++++++++++ .../test_tutorial_tile_change_detection.py | 27 +++++++++++++++++++ .../test_tutorial_tile_classification.py | 27 +++++++++++++++++++ .../test_tutorial_tile_completeness.py | 27 +++++++++++++++++++ 7 files changed, 111 insertions(+) create mode 100644 mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py create mode 100644 mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py create mode 100644 mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py create mode 100644 mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json index 4b857eaa0..8c0f817bd 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json @@ -4,6 +4,7 @@ "exampleImage2": "", "lookFor": "damaged buildings", "name": "change_detection_tutorial", + "tutorialDraftId": "test_tile_change_detection", "projectType": 3, "screens": [ null, diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json index b08c10dd7..0752c71a1 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json @@ -4,6 +4,7 @@ "exampleImage2": "https://firebasestorage.googleapis.com/v0/b/heigit-crowdmap.appspot.com/o/projectImages%2F1686065132355-tutorial-image-2-1x1.png?alt=media&token=bf8e67bc-d34c-4676-ba17-56bffc6b3f2d", "lookFor": "buildings", "name": "completeness_tutorial", + "tutorialDraftId": "test_tile_completeness", "projectType": 4, "screens": { "categories": { diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json index b8b31a9f9..b4e26e7bd 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json @@ -1,5 +1,6 @@ { "createdBy": "LtCUyou6CnSSc1H0Q0nDrN97x892", + "tutorialDraftId": "test_footprint_tutorial", "customOptions": [ { "description": "the shape does outline a building in the image", diff --git a/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py b/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py new file mode 100644 index 000000000..3d20b6289 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import FootprintTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_arbitrary_geometry_footprint_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "footprint.json") + ) + self.assertIsNotNone(FootprintTutorial(tutorial_draft=tutorial_draft)) + + def test_create_arbitrary_geometry_footprint_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "footprint.json") + ) + tutorial = FootprintTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py new file mode 100644 index 000000000..e394fd607 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import ChangeDetectionTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_tile_change_detection_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "change_detection.json") + ) + self.assertIsNotNone(ChangeDetectionTutorial(tutorial_draft=tutorial_draft)) + + def test_create_tile_change_detection_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "change_detection.json") + ) + tutorial = ChangeDetectionTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py new file mode 100644 index 000000000..5ba1c209a --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import ClassificationTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_tile_classification_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + ) + self.assertIsNotNone(ClassificationTutorial(tutorial_draft=tutorial_draft)) + + def test_create_tile_classification_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "tile_classification.json") + ) + tutorial = ClassificationTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py new file mode 100644 index 000000000..972c412ca --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import CompletenessTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_tile_completeness_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "completeness.json") + ) + self.assertIsNotNone(CompletenessTutorial(tutorial_draft=tutorial_draft)) + + def test_create_tile_completeness_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "completeness.json") + ) + tutorial = CompletenessTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() From 809f0f8f26b229d8ac9b9ca1784d11effff5e6c0 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 3 Mar 2025 15:26:38 +0100 Subject: [PATCH 106/107] fix: handle FutureWarning when concating dfs --- mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 97302b945..739ae39ae 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -143,6 +143,9 @@ def spatial_sampling(df, interval_length): if interval_length: sequence_df = filter_points(sequence_df, interval_length) + # below line prevents FutureWarning + # (https://stackoverflow.com/questions/73800841/add-series-as-a-new-row-into-dataframe-triggers-futurewarning) + sequence_df["is_pano"] = sequence_df["is_pano"].astype(bool) sampled_sequence_df = pd.concat([sampled_sequence_df, sequence_df], axis=0) # reverse order such that sequence are in direction of travel From a7319e490cc97193ebcd64cebcb549b5f0974ad9 Mon Sep 17 00:00:00 2001 From: Levi Szamek Date: Mon, 3 Mar 2025 16:16:31 +0100 Subject: [PATCH 107/107] fix: check if column exists before using astype --- .../mapswipe_workers/utils/spatial_sampling.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py index 739ae39ae..67f35c7e9 100644 --- a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -143,9 +143,10 @@ def spatial_sampling(df, interval_length): if interval_length: sequence_df = filter_points(sequence_df, interval_length) - # below line prevents FutureWarning - # (https://stackoverflow.com/questions/73800841/add-series-as-a-new-row-into-dataframe-triggers-futurewarning) - sequence_df["is_pano"] = sequence_df["is_pano"].astype(bool) + if "is_pano" in sequence_df.columns: + # below line prevents FutureWarning + # (https://stackoverflow.com/questions/73800841/add-series-as-a-new-row-into-dataframe-triggers-futurewarning) + sequence_df["is_pano"] = sequence_df["is_pano"].astype(bool) sampled_sequence_df = pd.concat([sampled_sequence_df, sequence_df], axis=0) # reverse order such that sequence are in direction of travel