Skip to content
This repository was archived by the owner on Apr 22, 2020. It is now read-only.

Commit 93f79ea

Browse files
mneedhammneedham
authored
Docs unified community detection (#807)
* Louvain - unified write result fields * LPA - unified write result fields * add write properties to louvain output * uniform ordering of results * pull 'write' up to the abstract class because all of them have that * add write properties for triangle count * fix the percentiles for triangle count - the result turned by the function is the size rather than needing to be computed via another computation * store write property for SCC * tidy up louvain order * add write properties to SCC docs * tidy up union find code * union find docs * triangle count docs * update label prop algo * lpa docs * Balanced Triads - use a default weight if no weight property passed in * Balanced Triads - compute percentiles for balanced triads
1 parent 023a400 commit 93f79ea

17 files changed

+545
-229
lines changed

‎algo/src/main/java/org/neo4j/graphalgo/BalancedTriadsProc.java‎

Lines changed: 83 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
*/
1919
package org.neo4j.graphalgo;
2020

21+
import com.carrotsearch.hppc.LongLongMap;
22+
import org.HdrHistogram.Histogram;
2123
import org.neo4j.graphalgo.api.HugeGraph;
2224
import org.neo4j.graphalgo.core.GraphLoader;
2325
import org.neo4j.graphalgo.core.ProcedureConfiguration;
@@ -26,7 +28,7 @@
2628
import org.neo4j.graphalgo.core.utils.paged.PagedAtomicIntegerArray;
2729
import org.neo4j.graphalgo.core.write.Exporter;
2830
import org.neo4j.graphalgo.impl.triangle.*;
29-
import org.neo4j.graphalgo.results.AbstractResultBuilder;
31+
import org.neo4j.graphalgo.results.AbstractCommunityResultBuilder;
3032
import org.neo4j.kernel.api.KernelTransaction;
3133
import org.neo4j.kernel.internal.GraphDatabaseAPI;
3234
import org.neo4j.logging.Log;
@@ -68,7 +70,7 @@ public Stream<HugeBalancedTriads.Result> balancedTriadsStream(
6870
final HugeGraph graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
6971
.withOptionalLabel(configuration.getNodeLabelOrQuery())
7072
.withOptionalRelationshipType(configuration.getRelationshipOrQuery())
71-
.withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
73+
.withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
7274
.withoutNodeWeights()
7375
.withSort(true)
7476
.withLog(log)
@@ -113,7 +115,7 @@ public Stream<Result> balancedTriads(
113115
graph = (HugeGraph) new GraphLoader(api, Pools.DEFAULT)
114116
.withOptionalLabel(configuration.getNodeLabelOrQuery())
115117
.withOptionalRelationshipType(configuration.getRelationshipOrQuery())
116-
.withRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
118+
.withOptionalRelationshipWeightsFromProperty(configuration.getWeightProperty(), 0.0)
117119
.withoutNodeWeights()
118120
.withSort(true)
119121
.withLog(log)
@@ -132,26 +134,33 @@ public Stream<Result> balancedTriads(
132134

133135
// write
134136
if (configuration.isWriteFlag()) {
137+
builder.withWrite(true);
138+
139+
String balancedProperty = configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY);
140+
builder.withBalancedProperty(balancedProperty);
141+
142+
String unbalancedProperty = configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY);
143+
builder.withUnbalancedProperty(unbalancedProperty);
144+
135145
try (ProgressTimer timer = builder.timeWrite()) {
136146
Exporter.of(api, graph)
137147
.withLog(log)
138148
.parallel(Pools.DEFAULT, configuration.getConcurrency(), terminationFlag)
139149
.build()
140150
.write(
141-
configuration.get("balancedProperty", DEFAULT_BALANCED_PROPERTY),
151+
balancedProperty,
142152
balancedTriads.getBalancedTriangles(),
143153
PagedAtomicIntegerArray.Translator.INSTANCE,
144-
configuration.get("unbalancedProperty", DEFAULT_UNBALANCED_PROPERTY),
154+
unbalancedProperty,
145155
balancedTriads.getUnbalancedTriangles(),
146156
PagedAtomicIntegerArray.Translator.INSTANCE);
147157
}
148158
}
149159

150160
// result
151-
return Stream.of(builder.withNodeCount(graph.nodeCount())
152-
.withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
153-
.withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount())
154-
.build());
161+
builder.withBalancedTriadCount(balancedTriads.getBalancedTriangleCount())
162+
.withUnbalancedTriadCount(balancedTriads.getUnbalancedTriangleCount());
163+
return Stream.of(builder.buildII(graph.nodeCount(), balancedTriads.getBalancedTriangles()::get));
155164
}
156165

157166
/**
@@ -162,54 +171,105 @@ public static class Result {
162171
public final long loadMillis;
163172
public final long computeMillis;
164173
public final long writeMillis;
174+
public final long postProcessingMillis;
175+
165176
public final long nodeCount;
166177
public final long balancedTriadCount;
167178
public final long unbalancedTriadCount;
168179

180+
public final long p1;
181+
public final long p5;
182+
public final long p10;
183+
public final long p25;
184+
public final long p50;
185+
public final long p75;
186+
public final long p90;
187+
public final long p95;
188+
public final long p99;
189+
public final long p100;
190+
191+
public final boolean write;
192+
public final String balancedProperty;
193+
public final String unbalancedProperty;
194+
195+
169196
public Result(
170197
long loadMillis,
171198
long computeMillis,
172199
long writeMillis,
173-
long nodeCount, long balancedTriadCount, long unbalancedTriadCount) {
200+
long postProcessingMillis,
201+
long nodeCount, long balancedTriadCount,
202+
long unbalancedTriadCount,
203+
long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1, boolean write, String balancedProperty, String unbalancedProperty) {
174204
this.loadMillis = loadMillis;
175205
this.computeMillis = computeMillis;
176206
this.writeMillis = writeMillis;
207+
this.postProcessingMillis = postProcessingMillis;
177208
this.nodeCount = nodeCount;
178209
this.balancedTriadCount = balancedTriadCount;
179210
this.unbalancedTriadCount = unbalancedTriadCount;
211+
this.p100 = p100;
212+
this.p99 = p99;
213+
this.p95 = p95;
214+
this.p90 = p90;
215+
this.p75 = p75;
216+
this.p50 = p50;
217+
this.p25 = p25;
218+
this.p10 = p10;
219+
this.p5 = p5;
220+
this.p1 = p1;
221+
this.write = write;
222+
this.balancedProperty = balancedProperty;
223+
this.unbalancedProperty = unbalancedProperty;
180224
}
181225
}
182226

183-
public class BalancedTriadsResultBuilder extends AbstractResultBuilder<Result> {
227+
public class BalancedTriadsResultBuilder extends AbstractCommunityResultBuilder<Result> {
184228

185-
private long nodeCount = 0;
186229
private long balancedTriadCount = 0;
187230
private long unbalancedTriadCount = 0;
231+
private String balancedProperty;
232+
private String unbalancedProperty;
188233

189-
public BalancedTriadsResultBuilder withNodeCount(long nodeCount) {
190-
this.nodeCount = nodeCount;
191-
return this;
192-
}
193234

194235
public BalancedTriadsResultBuilder withBalancedTriadCount(long balancedTriadCount) {
195236
this.balancedTriadCount = balancedTriadCount;
196237
return this;
197238
}
198239

240+
public BalancedTriadsResultBuilder withBalancedProperty(String property) {
241+
this.balancedProperty = property;
242+
return this;
243+
}
244+
245+
public BalancedTriadsResultBuilder withUnbalancedProperty(String property) {
246+
this.unbalancedProperty = property;
247+
return this;
248+
}
249+
199250
public BalancedTriadsResultBuilder withUnbalancedTriadCount(long unbalancedTriadCount) {
200251
this.unbalancedTriadCount = unbalancedTriadCount;
201252
return this;
202253
}
203254

204255
@Override
205-
public Result build() {
256+
protected Result build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
206257
return new Result(
207-
loadDuration,
208-
evalDuration,
209-
writeDuration,
210-
nodeCount,
211-
balancedTriadCount,
212-
unbalancedTriadCount);
258+
loadMillis, computeMillis, writeMillis, postProcessingMillis, nodeCount, balancedTriadCount, unbalancedTriadCount,
259+
communityHistogram.getValueAtPercentile(100),
260+
communityHistogram.getValueAtPercentile(99),
261+
communityHistogram.getValueAtPercentile(95),
262+
communityHistogram.getValueAtPercentile(90),
263+
communityHistogram.getValueAtPercentile(75),
264+
communityHistogram.getValueAtPercentile(50),
265+
communityHistogram.getValueAtPercentile(25),
266+
communityHistogram.getValueAtPercentile(10),
267+
communityHistogram.getValueAtPercentile(5),
268+
communityHistogram.getValueAtPercentile(1),
269+
write,
270+
balancedProperty,
271+
unbalancedProperty
272+
);
213273
}
214274
}
215275

‎algo/src/main/java/org/neo4j/graphalgo/LabelPropagationProc.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ public Stream<LabelPropagationStats> labelPropagation(
110110

111111
final int[] labels = compute(direction, iterations, batchSize, concurrency, graph, stats);
112112
if (configuration.isWriteFlag(DEFAULT_WRITE) && partitionProperty != null) {
113+
stats.withWrite(true);
113114
write(concurrency, partitionProperty, graph, labels, stats);
114115
}
115116

@@ -218,7 +219,6 @@ private void write(
218219
HeavyGraph graph,
219220
int[] labels,
220221
LabelPropagationStats.Builder stats) {
221-
stats.write(true);
222222
try (ProgressTimer timer = stats.timeWrite()) {
223223
Exporter.of(dbAPI, graph)
224224
.withLog(log)

‎algo/src/main/java/org/neo4j/graphalgo/LouvainProc.java‎

Lines changed: 64 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,18 @@ public Stream<LouvainResult> louvain(
105105
}
106106

107107
if (configuration.isWriteFlag()) {
108-
builder.timeWrite(() -> write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration));
108+
builder.timeWrite(() -> {
109+
String writeProperty = configuration.getWriteProperty("community");
110+
boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);
111+
String intermediateCommunitiesWriteProperty = configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities");
112+
113+
builder.withWrite(true);
114+
builder.withWriteProperty(writeProperty);
115+
builder.withIntermediateCommunities(includeIntermediateCommunities);
116+
builder.withIntermediateCommunitiesWriteProperty(intermediateCommunitiesWriteProperty);
117+
118+
write(graph, louvain.getDendrogram(), louvain.getCommunityIds(), configuration, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty);
119+
});
109120
}
110121

111122
builder.withIterations(louvain.getLevel());
@@ -170,18 +181,17 @@ public Graph graph(String label, String relationship, ProcedureConfiguration con
170181
.load(config.getGraphImpl());
171182
}
172183

173-
private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration) {
184+
private void write(Graph graph, int[][] allCommunities, int[] finalCommunities, ProcedureConfiguration configuration, String writeProperty, boolean includeIntermediateCommunities, String intermediateCommunitiesPropertyName) {
174185
log.debug("Writing results");
175-
boolean includeIntermediateCommunities = configuration.get(INCLUDE_INTERMEDIATE_COMMUNITIES, false);
176186

177187
new LouvainCommunityExporter(
178188
api,
179189
Pools.DEFAULT,
180190
configuration.getConcurrency(),
181191
graph,
182192
finalCommunities.length,
183-
configuration.getWriteProperty("community"),
184-
configuration.get(INTERMEDIATE_COMMUNITIES_WRITE_PROPERTY, "communities"))
193+
writeProperty,
194+
intermediateCommunitiesPropertyName)
185195
.export(allCommunities, finalCommunities, includeIntermediateCommunities);
186196
}
187197

@@ -205,29 +215,37 @@ public static class LouvainResult {
205215
-1,
206216
-1,
207217
0,
208-
new double[] {}, -1);
218+
new double[] {}, -1, false, null, false, null);
209219

210220
public final long loadMillis;
211221
public final long computeMillis;
212-
public final long postProcessingMillis;
213222
public final long writeMillis;
223+
public final long postProcessingMillis;
214224
public final long nodes;
215225
public final long communityCount;
216-
public final long p100;
217-
public final long p99;
218-
public final long p95;
219-
public final long p90;
220-
public final long p75;
221-
public final long p50;
222-
public final long p25;
223-
public final long p10;
224-
public final long p05;
225-
public final long p01;
226226
public final long iterations;
227227
public final List<Double> modularities;
228228
public final double modularity;
229-
230-
public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes, long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p05, long p01, long iterations, double[] modularities, double finalModularity) {
229+
public final long p1;
230+
public final long p5;
231+
public final long p10;
232+
public final long p25;
233+
public final long p50;
234+
public final long p75;
235+
public final long p90;
236+
public final long p95;
237+
public final long p99;
238+
public final long p100;
239+
public final boolean write;
240+
public final String writeProperty;
241+
public final boolean includeIntermediateCommunities;
242+
public final String intermediateCommunitiesWriteProperty;
243+
244+
public LouvainResult(long loadMillis, long computeMillis, long postProcessingMillis, long writeMillis, long nodes,
245+
long communityCount, long p100, long p99, long p95, long p90, long p75, long p50, long p25, long p10, long p5, long p1,
246+
long iterations, double[] modularities, double finalModularity,
247+
boolean write, String writeProperty,
248+
boolean includeIntermediateCommunities, String intermediateCommunitiesWriteProperty) {
231249
this.loadMillis = loadMillis;
232250
this.computeMillis = computeMillis;
233251
this.postProcessingMillis = postProcessingMillis;
@@ -242,12 +260,16 @@ public LouvainResult(long loadMillis, long computeMillis, long postProcessingMil
242260
this.p50 = p50;
243261
this.p25 = p25;
244262
this.p10 = p10;
245-
this.p05 = p05;
246-
this.p01 = p01;
263+
this.p5 = p5;
264+
this.p1 = p1;
247265
this.iterations = iterations;
248266
this.modularities = new ArrayList<>(modularities.length);
267+
this.write = write;
268+
this.includeIntermediateCommunities = includeIntermediateCommunities;
249269
for (double mod : modularities) this.modularities.add(mod);
250270
this.modularity = finalModularity;
271+
this.writeProperty = writeProperty;
272+
this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
251273
}
252274
}
253275

@@ -256,14 +278,22 @@ public static class Builder extends AbstractCommunityResultBuilder<LouvainResult
256278
private long iterations = -1;
257279
private double[] modularities = new double[] {};
258280
private double finalModularity = -1;
281+
private String writeProperty;
282+
private String intermediateCommunitiesWriteProperty;
283+
private boolean includeIntermediateCommunities;
284+
285+
public Builder withWriteProperty(String writeProperty) {
286+
this.writeProperty = writeProperty;
287+
return this;
288+
}
259289

260290
public Builder withIterations(long iterations) {
261291
this.iterations = iterations;
262292
return this;
263293
}
264294

265295
@Override
266-
protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram) {
296+
protected LouvainResult build(long loadMillis, long computeMillis, long writeMillis, long postProcessingMillis, long nodeCount, long communityCount, LongLongMap communitySizeMap, Histogram communityHistogram, boolean write) {
267297
return new LouvainResult(
268298
loadMillis,
269299
computeMillis,
@@ -281,7 +311,8 @@ protected LouvainResult build(long loadMillis, long computeMillis, long writeMil
281311
communityHistogram.getValueAtPercentile(10),
282312
communityHistogram.getValueAtPercentile(5),
283313
communityHistogram.getValueAtPercentile(1),
284-
iterations, modularities, finalModularity
314+
iterations, modularities, finalModularity,
315+
write, writeProperty, includeIntermediateCommunities, intermediateCommunitiesWriteProperty
285316
);
286317
}
287318

@@ -294,6 +325,16 @@ public Builder withFinalModularity(double finalModularity) {
294325
this.finalModularity = finalModularity;
295326
return null;
296327
}
328+
329+
public Builder withIntermediateCommunitiesWriteProperty(String intermediateCommunitiesWriteProperty) {
330+
this.intermediateCommunitiesWriteProperty = intermediateCommunitiesWriteProperty;
331+
return null;
332+
}
333+
334+
public Builder withIntermediateCommunities(boolean includeIntermediateCommunities) {
335+
this.includeIntermediateCommunities = includeIntermediateCommunities;
336+
return this;
337+
}
297338
}
298339

299340

0 commit comments

Comments
 (0)