From f06d7fedf5567c1c963cc1a7da4ff774746b7bfe Mon Sep 17 00:00:00 2001 From: Daniel Jilg Date: Fri, 28 Nov 2025 15:17:50 +0100 Subject: [PATCH 1/2] =?UTF-8?q?Add=20=E2=80=9Cin=E2=80=9D=20filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Query/CustomQuery+CompileDown.swift | 2 ++ .../DataTransferObjects/Query/Filter.swift | 35 +++++++++++++++---- Tests/QueryTests/FilterEqualsNullTests.swift | 25 +++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift b/Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift index b6b5bbb..2353386 100644 --- a/Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift +++ b/Sources/DataTransferObjects/Query/CustomQuery+CompileDown.swift @@ -162,6 +162,8 @@ public extension CustomQuery { return filter case .null: return filter + case .in: + return filter case .and(let filterExpression): return Filter.and(.init(fields: filterExpression.fields.map { compileRelativeFilterInterval(filter: $0) })) case .or(let filterExpression): diff --git a/Sources/DataTransferObjects/Query/Filter.swift b/Sources/DataTransferObjects/Query/Filter.swift index 7d8a7ba..8453410 100644 --- a/Sources/DataTransferObjects/Query/Filter.swift +++ b/Sources/DataTransferObjects/Query/Filter.swift @@ -193,22 +193,34 @@ extension FilterEquals.MatchValue: Codable { var container = encoder.singleValueContainer() switch self { - case .string(let value): + case let .string(value): try container.encode(value) - case .int(let value): + case let .int(value): try container.encode(value) - case .double(let value): + case let .double(value): try container.encode(value) - case .arrayString(let value): + case let .arrayString(value): try container.encode(value) - case .arrayInt(let value): + case let .arrayInt(value): try container.encode(value) - case .arrayDouble(let value): + case let .arrayDouble(value): try container.encode(value) } } } +/// The in filter can match input rows against a set of values, where a match occurs if the value is contained in the set. +public struct FilterIn: Codable, Hashable, Equatable, Sendable { + public init(dimension: String, values: [String]) { + self.dimension = dimension + self.values = values + } + + public let dimension: String + public let values: [String] + // extractionFn not supported atm +} + /// The null filter matches rows where a column value is null. public struct FilterNull: Codable, Hashable, Equatable, Sendable { public init(column: String) { @@ -223,7 +235,7 @@ public struct FilterNull: Codable, Hashable, Equatable, Sendable { public indirect enum Filter: Codable, Hashable, Equatable, Sendable { /// The selector filter will match a specific dimension with a specific value. /// Selector filters can be used as the base filters for more complex Boolean - /// expressions of filters. + /// expressions of filters (deprecated -- use equals instead) case selector(FilterSelector) /// The column comparison filter is similar to the selector filter, but instead @@ -252,6 +264,10 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable { /// The null filter matches rows where a column value is null. case null(FilterNull) + /// The in filter can match input rows against a set of values, where a match occurs + /// if the value is contained in the set. + case `in`(FilterIn) + // logical expression filters case and(FilterExpression) case or(FilterExpression) @@ -280,6 +296,8 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable { self = try .equals(FilterEquals(from: decoder)) case "null": self = try .null(FilterNull(from: decoder)) + case "in": + self = try .in(FilterIn(from: decoder)) case "and": self = try .and(FilterExpression(from: decoder)) case "or": @@ -321,6 +339,9 @@ public indirect enum Filter: Codable, Hashable, Equatable, Sendable { case let .null(null): try container.encode("null", forKey: .type) try null.encode(to: encoder) + case let .in(inFilter): + try container.encode("in", forKey: .type) + try inFilter.self.encode(to: encoder) case let .and(and): try container.encode("and", forKey: .type) try and.encode(to: encoder) diff --git a/Tests/QueryTests/FilterEqualsNullTests.swift b/Tests/QueryTests/FilterEqualsNullTests.swift index 7e45e40..f688342 100644 --- a/Tests/QueryTests/FilterEqualsNullTests.swift +++ b/Tests/QueryTests/FilterEqualsNullTests.swift @@ -205,4 +205,29 @@ class FilterEqualsNullTests: XCTestCase { XCTAssertEqual(filterNull, decodedFilter) XCTAssertEqual(filterJSON, String(data: encodedFilter, encoding: .utf8)) } + + func testFilterIn() throws { + let filterJSON = """ + { + "dimension": "outlaw", + "type": "in", + "values": ["Good", "Bad", "Ugly"] + } + """ + .filter { !$0.isWhitespace } + + let filterNull = Filter.in( + FilterIn(dimension: "outlaw", values: ["Good", "Bad", "Ugly"]) + ) + + let decodedFilter = try JSONDecoder.telemetryDecoder.decode( + Filter.self, + from: filterJSON.data(using: .utf8)! + ) + + let encodedFilter = try JSONEncoder.telemetryEncoder.encode(filterNull) + + XCTAssertEqual(filterNull, decodedFilter) + XCTAssertEqual(filterJSON, String(data: encodedFilter, encoding: .utf8)) + } } From e65b8774e06fa68be17ccf0b4f3b167eb0e41acc Mon Sep 17 00:00:00 2001 From: Daniel Jilg Date: Fri, 28 Nov 2025 15:21:07 +0100 Subject: [PATCH 2/2] Fix linter warnings --- .../CustomQuery+Retention.swift | 54 +++++++++---------- .../RetentionQueryGenerationTests.swift | 36 ++++++------- .../ScanQueryResultTests.swift | 4 +- Tests/QueryTests/AggregatorTests.swift | 1 - 4 files changed, 47 insertions(+), 48 deletions(-) diff --git a/Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift b/Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift index 8a24169..8e94068 100644 --- a/Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift +++ b/Sources/DataTransferObjects/QueryGeneration/CustomQuery+Retention.swift @@ -4,31 +4,31 @@ import DateOperations extension CustomQuery { func precompiledRetentionQuery() throws -> CustomQuery { var query = self - + // Get the query intervals - we need at least one interval guard let queryIntervals = intervals ?? relativeIntervals?.map({ QueryTimeInterval.from(relativeTimeInterval: $0) }), let firstInterval = queryIntervals.first else { throw QueryGenerationError.keyMissing(reason: "Missing intervals for retention query") } - + let beginDate = firstInterval.beginningDate let endDate = firstInterval.endDate - + // Use the query's granularity to determine retention period, defaulting to month if not specified let retentionGranularity = query.granularity ?? .month - + // Validate minimum interval based on granularity try validateMinimumInterval(from: beginDate, to: endDate, granularity: retentionGranularity) - + // Split into intervals based on the specified granularity let retentionIntervals = try splitIntoIntervals(from: beginDate, to: endDate, granularity: retentionGranularity) - + // Generate Aggregators var aggregators = [Aggregator]() for interval in retentionIntervals { aggregators.append(aggregator(for: interval)) } - + // Generate Post-Aggregators var postAggregators = [PostAggregator]() for row in retentionIntervals { @@ -36,26 +36,26 @@ extension CustomQuery { postAggregators.append(postAggregatorBetween(interval1: row, interval2: column)) } } - + // Set the query properties query.queryType = .groupBy query.granularity = .all query.aggregations = uniqued(aggregators) query.postAggregations = uniqued(postAggregators) - + return query } - + private func uniqued(_ array: [T]) -> [T] { var set = Set() return array.filter { set.insert($0).inserted } } - + // MARK: - Helper Methods - + private func validateMinimumInterval(from beginDate: Date, to endDate: Date, granularity: QueryGranularity) throws { let calendar = Calendar.current - + switch granularity { case .day: let components = calendar.dateComponents([.day], from: beginDate, to: endDate) @@ -86,11 +86,11 @@ extension CustomQuery { throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity") } } - + private func splitIntoIntervals(from fromDate: Date, to toDate: Date, granularity: QueryGranularity) throws -> [DateInterval] { let calendar = Calendar.current var intervals = [DateInterval]() - + switch granularity { case .day: let numberOfDays = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .day) @@ -100,7 +100,7 @@ extension CustomQuery { let endOfDay = startOfDay.end(of: .day) ?? startOfDay intervals.append(DateInterval(start: startOfDay, end: endOfDay)) } - + case .week: let numberOfWeeks = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .weekOfYear) for week in 0...numberOfWeeks { @@ -109,7 +109,7 @@ extension CustomQuery { let endOfWeek = startOfWeek.end(of: .weekOfYear) ?? startOfWeek intervals.append(DateInterval(start: startOfWeek, end: endOfWeek)) } - + case .month: let numberOfMonths = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .month) for month in 0...numberOfMonths { @@ -118,7 +118,7 @@ extension CustomQuery { let endOfMonth = startOfMonth.end(of: .month) ?? startOfMonth intervals.append(DateInterval(start: startOfMonth, end: endOfMonth)) } - + case .quarter: let numberOfQuarters = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .quarter) for quarter in 0...numberOfQuarters { @@ -127,7 +127,7 @@ extension CustomQuery { let endOfQuarter = startOfQuarter.end(of: .quarter) ?? startOfQuarter intervals.append(DateInterval(start: startOfQuarter, end: endOfQuarter)) } - + case .year: let numberOfYears = numberOfUnitsBetween(beginDate: fromDate, endDate: toDate, component: .year) for year in 0...numberOfYears { @@ -136,18 +136,18 @@ extension CustomQuery { let endOfYear = startOfYear.end(of: .year) ?? startOfYear intervals.append(DateInterval(start: startOfYear, end: endOfYear)) } - + default: throw QueryGenerationError.notImplemented(reason: "Retention queries support day, week, month, quarter, or year granularity") } - + return intervals } - + private func numberOfUnitsBetween(beginDate: Date, endDate: Date, component: Calendar.Component) -> Int { let calendar = Calendar.current let components = calendar.dateComponents([component], from: beginDate, to: endDate) - + switch component { case .day: return components.day ?? 0 @@ -163,13 +163,13 @@ extension CustomQuery { return 0 } } - + private func title(for interval: DateInterval) -> String { let formatter = ISO8601DateFormatter() formatter.formatOptions = [.withFullDate] return "\(formatter.string(from: interval.start))_\(formatter.string(from: interval.end))" } - + private func aggregator(for interval: DateInterval) -> Aggregator { .filtered(.init( filter: .interval(.init( @@ -182,7 +182,7 @@ extension CustomQuery { )) )) } - + private func postAggregatorBetween(interval1: DateInterval, interval2: DateInterval) -> PostAggregator { .thetaSketchEstimate(.init( name: "retention_\(title(for: interval1))_\(title(for: interval2))", @@ -195,4 +195,4 @@ extension CustomQuery { )) )) } -} \ No newline at end of file +} diff --git a/Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift b/Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift index 31f9da1..ada36e9 100644 --- a/Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift +++ b/Tests/QueryGenerationTests/RetentionQueryGenerationTests.swift @@ -129,7 +129,7 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .month ) XCTAssertThrowsError(try monthQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + let monthQuery2 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -137,7 +137,7 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .month ) XCTAssertThrowsError(try monthQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + let monthQuery3 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -145,12 +145,12 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .month ) XCTAssertNoThrow(try monthQuery3.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + // Test daily retention let startDate = Date(iso8601String: "2022-08-01T00:00:00.000Z")! let sameDay = Date(iso8601String: "2022-08-01T12:00:00.000Z")! let nextDay = Date(iso8601String: "2022-08-02T00:00:00.000Z")! - + let dayQuery1 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -158,7 +158,7 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .day ) XCTAssertThrowsError(try dayQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + let dayQuery2 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -166,12 +166,12 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .day ) XCTAssertNoThrow(try dayQuery2.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + // Test weekly retention let weekStart = Date(iso8601String: "2022-08-01T00:00:00.000Z")! let weekMid = Date(iso8601String: "2022-08-05T00:00:00.000Z")! let weekEnd = Date(iso8601String: "2022-08-08T00:00:00.000Z")! - + let weekQuery1 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -179,7 +179,7 @@ final class RetentionQueryGenerationTests: XCTestCase { granularity: .week ) XCTAssertThrowsError(try weekQuery1.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [UUID()], isSuperOrg: false)) - + let weekQuery2 = CustomQuery( queryType: .retention, dataSource: "com.telemetrydeck.all", @@ -204,22 +204,22 @@ final class RetentionQueryGenerationTests: XCTestCase { )], granularity: .month // Explicitly set to month ) - + let compiledQuery = try query.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true) - + // Verify the compiled query has the expected structure XCTAssertEqual(compiledQuery.queryType, .groupBy) XCTAssertEqual(compiledQuery.granularity, .all) XCTAssertNotNil(compiledQuery.aggregations) XCTAssertNotNil(compiledQuery.postAggregations) - + // The generated query should match the expected structure from tinyQuery // (though the exact aggregator names might differ due to date formatting) } - + func testRetentionWithDifferentGranularities() throws { let appID = UUID(uuidString: "79167A27-EBBF-4012-9974-160624E5D07B")! - + // Test daily retention - 7 days should generate 8 intervals (0-7 inclusive) let dailyQuery = CustomQuery( queryType: .retention, @@ -233,12 +233,12 @@ final class RetentionQueryGenerationTests: XCTestCase { )], granularity: .day ) - + let compiledDailyQuery = try dailyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true) XCTAssertEqual(compiledDailyQuery.aggregations?.count, 7) // 7 days // Post-aggregations should be n*(n+1)/2 for n intervals XCTAssertEqual(compiledDailyQuery.postAggregations?.count, 28) // 7*8/2 = 28 - + // Test weekly retention - 4 weeks let weeklyQuery = CustomQuery( queryType: .retention, @@ -252,11 +252,11 @@ final class RetentionQueryGenerationTests: XCTestCase { )], granularity: .week ) - + let compiledWeeklyQuery = try weeklyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true) XCTAssertEqual(compiledWeeklyQuery.aggregations?.count, 5) // 5 weeks (spans into 5th week) XCTAssertEqual(compiledWeeklyQuery.postAggregations?.count, 15) // 5*6/2 = 15 - + // Test monthly retention - 3 months let monthlyQuery = CustomQuery( queryType: .retention, @@ -270,7 +270,7 @@ final class RetentionQueryGenerationTests: XCTestCase { )], granularity: .month ) - + let compiledMonthlyQuery = try monthlyQuery.precompile(namespace: nil, useNamespace: false, organizationAppIDs: [appID], isSuperOrg: true) XCTAssertEqual(compiledMonthlyQuery.aggregations?.count, 3) // 3 months XCTAssertEqual(compiledMonthlyQuery.postAggregations?.count, 6) // 3*4/2 = 6 diff --git a/Tests/QueryResultTests/ScanQueryResultTests.swift b/Tests/QueryResultTests/ScanQueryResultTests.swift index 5767c3e..0c7219f 100644 --- a/Tests/QueryResultTests/ScanQueryResultTests.swift +++ b/Tests/QueryResultTests/ScanQueryResultTests.swift @@ -52,8 +52,8 @@ class ScanQueryResultTests: XCTestCase { "events": [{ "__time": 1741168800000, "payload": [ - "TelemetryDeck.API.Ingest.version:v2", - "TelemetryDeck.Accessibility.isInvertColorsEnabled:false", + "TelemetryDeck.API.Ingest.version:v2", + "TelemetryDeck.Accessibility.isInvertColorsEnabled:false", "TelemetryDeck.Accessibility.isReduceMotionEnabled:false" ] }], diff --git a/Tests/QueryTests/AggregatorTests.swift b/Tests/QueryTests/AggregatorTests.swift index a0bfd03..c82cdbd 100644 --- a/Tests/QueryTests/AggregatorTests.swift +++ b/Tests/QueryTests/AggregatorTests.swift @@ -252,7 +252,6 @@ final class AggregatorTests: XCTestCase { XCTAssertEqual(String(data: encodedAggregators, encoding: .utf8)!, expectedEncodedAggregators) } - func testUserCountAggregator() throws { let stringRepresentation = """ [