Skip to content

Fix Data Browser bugs for large datasets and query handling #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mrjeremystookey wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix Data Browser bugs for large datasets and query handling #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 73 additions & 45 deletions DittoToolsAndroid/src/main/java/live/ditto/tools/databrowser/Documents.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,15 @@ import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.text.BasicTextField
import androidx.compose.material3.*
import androidx.compose.material3.Card
import androidx.compose.material3.CardDefaults
import androidx.compose.material3.Divider
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Search
import androidx.compose.runtime.*
Expand Down Expand Up @@ -37,9 +45,19 @@ fun Documents(collectionName: String, isStandAlone: Boolean) {

val selectedDoc by viewModel.selectedDoc.observeAsState()
val docsList by viewModel.docsList.observeAsState()
val errorMessage by viewModel.errorMessage.observeAsState()
var selectedIndex by remember { mutableStateOf(0) }
var startUp by remember { mutableStateOf(true) }

// Auto-select first document when docsList loads/changes
LaunchedEffect(docsList) {
if (!docsList.isNullOrEmpty() && startUp) {
selectedIndex = 0
viewModel.selectedDoc.value = docsList!![0]
startUp = false
}
}

Column(
modifier = Modifier
.fillMaxWidth()
Expand All @@ -56,70 +74,80 @@ fun Documents(collectionName: String, isStandAlone: Boolean) {
selectedIndex = 0
})
Spacer(modifier = Modifier.height(16.dp))
Text(text = "Docs count: ${docsList?.size}")

errorMessage?.let { error ->
Card(
modifier = Modifier.fillMaxWidth(),
colors = CardDefaults.cardColors(
containerColor = MaterialTheme.colorScheme.errorContainer
)
) {
Text(
text = error,
color = MaterialTheme.colorScheme.onErrorContainer,
modifier = Modifier.padding(16.dp)
)
}
Spacer(modifier = Modifier.height(16.dp))
}

Text(text = "Docs count: ${docsList?.size ?: "Loading..."}")
Spacer(modifier = Modifier.height(16.dp))

Row {
Text(
text = "Doc ID: ",
textAlign = TextAlign.Start,
modifier = Modifier
.clickable {

}
textAlign = TextAlign.Start
)

if (!docsList.isNullOrEmpty()) {
Box {
// Show selected item or "select" if no item is selected
(if ((startUp)) "select" else docsList?.get(selectedIndex)?.id)?.let {
val isLargeDataset = (docsList?.size ?: 0) > 1000

if (isLargeDataset) {
// For large datasets, show ID without dropdown to prevent OutOfMemoryError
docsList?.getOrNull(selectedIndex)?.id?.let { docId ->
Text(
text = it,
text = docId,
textAlign = TextAlign.Start,
color = Color.Blue,
modifier = Modifier
.clickable {
showMenu = true

startUp = false
}
color = MaterialTheme.colorScheme.primary
)
}

// Dropdown menu
DropdownMenu(
expanded = showMenu,
onDismissRequest = { showMenu = false }
) {
docsList?.forEachIndexed { index, item ->
DropdownMenuItem(onClick = {
selectedIndex = index
viewModel.selectedDoc.value = item
}, text = {
Text(text = item.id)
}, modifier = Modifier.onKeyEvent { keyEvent ->
when (keyEvent.key) {
Key.Spacebar -> {
when (keyEvent.type) {
KeyEventType.KeyUp -> {
selectedIndex = index
viewModel.selectedDoc.value = item
true
}
else -> false
}
} else {
// For small datasets, show dropdown
Box {
docsList?.getOrNull(selectedIndex)?.id?.let { docId ->
Text(
text = docId,
textAlign = TextAlign.Start,
color = Color.Blue,
modifier = Modifier
.clickable {
showMenu = true
}
else -> false
}
})
)
}

DropdownMenu(
expanded = showMenu,
onDismissRequest = { showMenu = false }
) {
docsList?.forEachIndexed { index, item ->
DropdownMenuItem(onClick = {
selectedIndex = index
viewModel.selectedDoc.value = item
showMenu = false
}, text = {
Text(text = item.id)
})
}
}
}
}
} else {
Text(
text = "No Docs",
textAlign = TextAlign.Start,
color = Color.Blue,
color = MaterialTheme.colorScheme.primary
)
}
}
Expand Down
81 changes: 64 additions & 17 deletions DittoToolsAndroid/src/main/java/live/ditto/tools/databrowser/DocumentsViewModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,62 +9,109 @@ class DocumentsViewModel(private val collectionName: String, isStandAlone: Boole
val docsList: MutableLiveData<MutableList<Document>> = MutableLiveData<MutableList<Document>>(mutableListOf())
var docProperties: MutableLiveData<List<String>> = MutableLiveData(emptyList())
var selectedDoc = MutableLiveData<Document>()
var errorMessage = MutableLiveData<String?>()

// Store all documents for client-side filtering
private var allDocuments: MutableList<Document> = mutableListOf()
private var currentFilter: String = ""
private var isDQLMode: Boolean = false

val subscription = if (isStandAlone) DittoHandler.ditto.store.collection(collectionName).findAll().limit(1000).subscribe() else null
private var liveQuery = DittoHandler.ditto.store.collection(collectionName).findAll().limit(1000).observeLocal { docs, _ ->

docsList.value?.clear()
val newDocsList = mutableListOf<Document>()
for(doc in docs) {
this.docProperties.postValue(doc.value.keys.map{it}.sorted())

val docValues = mutableMapOf<String, Any?>()
for((key, value) in doc.value) {
docValues[key] = value
}
docsList.value?.add(Document(doc.id.toString(), docValues))
newDocsList.add(Document(doc.id.toString(), docValues))
}
allDocuments = newDocsList
applyFilter()
}

private fun findAllLiveQuery() {
this.liveQuery = DittoHandler.ditto.store.collection(collectionName).findAll().limit(1000).observeLocal { docs, _ ->
docsList.value?.clear()
val newDocsList = mutableListOf<Document>()
for(doc in docs) {
this.docProperties.postValue(doc.value.keys.map{it}.sorted())

val docValues = mutableMapOf<String, Any?>()
for((key, value) in doc.value) {
docValues[key] = value
}
docsList.value?.add(Document(doc.id.toString(), docValues))
newDocsList.add(Document(doc.id.toString(), docValues))
}
allDocuments = newDocsList
applyFilter()
}
}

private fun findWithFilterLiveQuery(queryString: String) {
this.liveQuery = DittoHandler.ditto.store.collection(collectionName).find(queryString).limit(1000).observeLocal { docs, _ ->
docsList.value?.clear()
try {
errorMessage.postValue(null)
this.liveQuery = DittoHandler.ditto.store.collection(collectionName).find(queryString).limit(1000).observeLocal { docs, _ ->
val newDocsList = mutableListOf<Document>()

for(doc in docs) {
this.docProperties.postValue(doc.value.keys.map{it}.sorted())
for(doc in docs) {
this.docProperties.postValue(doc.value.keys.map{it}.sorted())

val docValues = mutableMapOf<String, Any?>()
for((key, value) in doc.value) {
docValues[key] = value
val docValues = mutableMapOf<String, Any?>()
for((key, value) in doc.value) {
docValues[key] = value
}
newDocsList.add(Document(doc.id.toString(), docValues))
}
docsList.value?.add(Document(doc.id.toString(), docValues))
docsList.postValue(newDocsList)
}
} catch (e: Exception) {
errorMessage.postValue("Invalid DQL query: ${e.message}")
docsList.postValue(mutableListOf())
}
}

fun filterDocs(queryString: String) {
liveQuery.close()
currentFilter = queryString

if(queryString.isEmpty()) {
findAllLiveQuery()
}
else {
if (isDQLQuery(queryString)) {
// User provided explicit DQL query - use server-side filtering
liveQuery.close()
isDQLMode = true
findWithFilterLiveQuery(queryString)
} else {
// Simple text search - use client-side filtering
if (isDQLMode) {
// Switching from DQL mode back to simple search
// Need to restart the findAll query
liveQuery.close()
isDQLMode = false
findAllLiveQuery()
} else {
// Already in simple mode, just filter the existing data
applyFilter()
}
}
}

private fun applyFilter() {
val filtered = if (currentFilter.isEmpty()) {
allDocuments
} else {
// Filter documents where ID contains the search text (case-insensitive)
allDocuments.filter { doc ->
doc.id.contains(currentFilter, ignoreCase = true)
}.toMutableList()
}
docsList.postValue(filtered)
}

private fun isDQLQuery(text: String): Boolean {
// Check if the text contains DQL operators
val dqlOperators = listOf("==", "!=", "CONTAINS", "contains", ">", "<", ">=", "<=", "AND", "and", "OR", "or", "IN", "in")
return dqlOperators.any { text.contains(it) }
}

class MyViewModelFactory(private val collectionName: String, private val isStandAlone: Boolean) :
Expand Down