From 463799c7a09c81f1ffb58e577343201db0dc97fe Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Fri, 3 Jan 2025 11:51:45 +0100 Subject: [PATCH 1/9] Create update.sh Update script that WorksForMe* Might be a starting point for someone to develop a more useful general update solution. --- update.sh | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 update.sh diff --git a/update.sh b/update.sh new file mode 100644 index 000000000..1e7e81f15 --- /dev/null +++ b/update.sh @@ -0,0 +1,164 @@ +##!/bin/bash +set -euo pipefail + +### constants ### +FILE="Part-DB-release-stage" +GITREPO="/opt/partdb/git-repo" +PRODENV="/var/www/Part-DB-server" +BACKUPDIR="/HDD5/backups/partdb" +WEB_AVAILABLE="/etc/apache2/sites-available" +WEB_ENABLED="/etc/apache2/sites-enabled" +WEBSERVICE="apache2" +LIVE_CONF="01_partdb.conf" +TEST_CONF="06_partdb_testing.conf" +MNT_CONF="07_partdb_maintenance.conf" + +# We should probably do a sanity check here? +DBUSER="$1" +DBPASSWD="$2" + +### variables ### +stage="1" + +echo "" +echo "*** Part-DB upgrade tool v0.0001 (WorksForMe* edition) ***" + + +if [ $(whoami) != 'root' ]; then + echo "" + echo "This script must be run as root!" + exit -1 +fi + +if [ -f $FILE ]; then + stage=$(<$FILE) +fi + +if [ "$stage" = "1" ]; then # no update currently in progress + cd $GITREPO + git fetch --tags + # get latest version + gitver=$(git describe --tags --abbrev=0) + currentver=$( $FILE + else + echo "* Invalid stage: $stage, expected 1" + exit 1 + fi + ;; + "2") + if [ "$stage" = "2" ]; then + echo "* Stage 2: Dump DB and update Part-DB via git" + # cd into working dir + cd $GITREPO + git fetch --tags + # get latest version + version=$(git describe --tags --abbrev=0) + # dump DB, preventing overwrite by re-execution if e.g. the migration broke the database structure + mysqldump -u$DBUSER -p$DBPASSWD partdb > $BACKUPDIR/partdb_before_update_$version_$(date -Iseconds).sql + # pull changes, checkout latest tag + git pull && git checkout $version + # copy config and media files and correct ownership + cp "$PRODENV/.env.local" $GITREPO + cp -rn $PRODENV/public/media/ $GITREPO/public/ + chown -R www-data:www-data $GITREPO + # merge .env with .env.local, config/services.yaml, config/parameters.yaml if changed + # TODO how to handle customizations ??? meld ??? + echo "* Files are in place, build step pending" + echo "3" > $FILE + else + echo "* Invalid stage: $stage, expected 2" + exit 2 + fi + ;; + "3") + if [ "$stage" = "3" ] ; then + echo "* Stage 3: Build process" + # build steps + cd $GITREPO + environment=$(sed -nr 's/APP_ENV=(.*)/\1/p' .env.local) + if [ environment != "dev" ]; then + environment="no-dev" + fi + sudo -u www-data composer install --$environment -o + yarn install + yarn build + # check if installation succeeded and migrate db + sudo -u www-data php bin/console partdb:check-requirements + sudo -u www-data php bin/console doctrine:migrations:migrate + sudo -u www-data php bin/console cache:clear + # we can mess with the production db because we have a very recent backup + rsync -av --exclude=$GITREPO/.git* $GITREPO/ $PRODENV-test + + echo "* The new Part-DB version can now be tested. You may need to merge .env.local with .env and check yaml files in config/." + echo "4" > $FILE + else + echo "* Invalid stage: $stage, expected 3" + exit 3 + fi + ;; + "4") + if [ "$stage" = "4" ]; then + echo "Stage 4: Put Part-DB back in production mode, retaining the old copy" + # copy all to prod environment + mv $PRODENV $PRODENV-old + mv $PRODENV-test $PRODENV + # remove link to maintenance PartDB VHost + rm $WEB_ENABLED/$LIVE_CONF $WEB_ENABLED/$TEST_CONF + # link the new partdb version + ln -sf $WEB_AVAILABLE/$LIVE_CONF $WEB_ENABLED/$LIVE_CONF + # reload apache + if ! [ `systemctl reload $WEBSERVICE && systemctl is-active --quiet $WEBSERVICE` ]; then + echo "* Webserver restart failed! Please check your $WEBSERVICE site configurations." + break + fi + echo "" + echo "*** Done. ***" + rm $FILE + else + echo "* Invalid stage: $stage, expected 4" + exit 4 + fi + ;; + esac + if [ -f $FILE ]; then + stage=$(<$FILE) + else + stage="1" + fi + else + echo "Update process aborted before stage $curstage." + break + fi +done +exit 0 From 6b4ff26a6354188806f574bc3f75622f24be8953 Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Wed, 28 May 2025 12:34:10 +0200 Subject: [PATCH 2/9] More generous search Split the keyword into tokens (at spaces). Search for every token individually, then combine results with AND. --- src/DataTables/Filters/PartSearchFilter.php | 60 +++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index 6e2e58949..1be5fed1a 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -65,11 +65,16 @@ class PartSearchFilter implements FilterInterface /** @var bool Use Internal Part number for searching */ protected bool $ipn = true; + /** @var int Helper variable for hacky array_map variable injection */ + protected int $it = 0; + public function __construct( /** @var string The string to query for */ protected string $keyword ) { + // Transform keyword and trim excess spaces + $keyword = trim(str_replace('+', ' ', $keyword)); } protected function getFieldsToSearch(): array @@ -125,26 +130,50 @@ public function apply(QueryBuilder $queryBuilder): void return; } - //Convert the fields to search to a list of expressions - $expressions = array_map(function (string $field): string { - if ($this->regex) { + if($this->regex) { + //Convert the fields to search to a list of expressions + $expressions = array_map(function (string $field): string { return sprintf("REGEXP(%s, :search_query) = TRUE", $field); - } - - return sprintf("ILIKE(%s, :search_query) = TRUE", $field); - }, $fields_to_search); + }, $fields_to_search); - //Add Or concatenation of the expressions to our query - $queryBuilder->andWhere( - $queryBuilder->expr()->orX(...$expressions) - ); + //Add Or concatenation of the expressions to our query + $queryBuilder->andWhere( + $queryBuilder->expr()->orX(...$expressions) + ); - //For regex, we pass the query as is, for like we add % to the start and end as wildcards - if ($this->regex) { + //For regex, we pass the query as is, save html special chars $queryBuilder->setParameter('search_query', $this->keyword); - } else { - $queryBuilder->setParameter('search_query', '%' . $this->keyword . '%'); + return; } + + //Split keyword on spaces, but limit token count to not blow up the DB + $tokens = explode(' ', $this->keyword, 5); + + $params = new \Doctrine\Common\Collections\ArrayCollection(); + + //Perform search of every single token in every selected field, AND the where clauses + for ($i = 0; $i < sizeof($tokens); $i++) { + $this->it = $i; + $tokens[$i] = trim($tokens[$i]); + + //Skip empty words (e.g. because of multiple spaces) + if ($tokens[$i] === '') continue; + + //Convert the fields to search to a list of expressions + $expressions = array_map(function (string $field): string { + return sprintf("ILIKE(%s, :search_query%u) = TRUE", $field, $this->it); + }, $fields_to_search); + + //Aggregate the parameters for consolidated commission + $params[] = new \Doctrine\ORM\Query\Parameter('search_query' . $i, '%' . $tokens[$i] . '%'); + + //Add Or concatenation of the expressions to our query + $queryBuilder->andWhere( + $queryBuilder->expr()->orX(...$expressions) + ); + } + $queryBuilder->setParameters($params); + } public function getKeyword(): string @@ -301,5 +330,4 @@ public function setComment(bool $comment): PartSearchFilter return $this; } - } From 8952116af422cd47a627ffe463ab525dfe4659a5 Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Fri, 3 Jan 2025 11:51:45 +0100 Subject: [PATCH 3/9] Create update.sh Update script that WorksForMe* Might be a starting point for someone to develop a more useful general update solution. --- update.sh | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 update.sh diff --git a/update.sh b/update.sh new file mode 100644 index 000000000..1e7e81f15 --- /dev/null +++ b/update.sh @@ -0,0 +1,164 @@ +##!/bin/bash +set -euo pipefail + +### constants ### +FILE="Part-DB-release-stage" +GITREPO="/opt/partdb/git-repo" +PRODENV="/var/www/Part-DB-server" +BACKUPDIR="/HDD5/backups/partdb" +WEB_AVAILABLE="/etc/apache2/sites-available" +WEB_ENABLED="/etc/apache2/sites-enabled" +WEBSERVICE="apache2" +LIVE_CONF="01_partdb.conf" +TEST_CONF="06_partdb_testing.conf" +MNT_CONF="07_partdb_maintenance.conf" + +# We should probably do a sanity check here? +DBUSER="$1" +DBPASSWD="$2" + +### variables ### +stage="1" + +echo "" +echo "*** Part-DB upgrade tool v0.0001 (WorksForMe* edition) ***" + + +if [ $(whoami) != 'root' ]; then + echo "" + echo "This script must be run as root!" + exit -1 +fi + +if [ -f $FILE ]; then + stage=$(<$FILE) +fi + +if [ "$stage" = "1" ]; then # no update currently in progress + cd $GITREPO + git fetch --tags + # get latest version + gitver=$(git describe --tags --abbrev=0) + currentver=$( $FILE + else + echo "* Invalid stage: $stage, expected 1" + exit 1 + fi + ;; + "2") + if [ "$stage" = "2" ]; then + echo "* Stage 2: Dump DB and update Part-DB via git" + # cd into working dir + cd $GITREPO + git fetch --tags + # get latest version + version=$(git describe --tags --abbrev=0) + # dump DB, preventing overwrite by re-execution if e.g. the migration broke the database structure + mysqldump -u$DBUSER -p$DBPASSWD partdb > $BACKUPDIR/partdb_before_update_$version_$(date -Iseconds).sql + # pull changes, checkout latest tag + git pull && git checkout $version + # copy config and media files and correct ownership + cp "$PRODENV/.env.local" $GITREPO + cp -rn $PRODENV/public/media/ $GITREPO/public/ + chown -R www-data:www-data $GITREPO + # merge .env with .env.local, config/services.yaml, config/parameters.yaml if changed + # TODO how to handle customizations ??? meld ??? + echo "* Files are in place, build step pending" + echo "3" > $FILE + else + echo "* Invalid stage: $stage, expected 2" + exit 2 + fi + ;; + "3") + if [ "$stage" = "3" ] ; then + echo "* Stage 3: Build process" + # build steps + cd $GITREPO + environment=$(sed -nr 's/APP_ENV=(.*)/\1/p' .env.local) + if [ environment != "dev" ]; then + environment="no-dev" + fi + sudo -u www-data composer install --$environment -o + yarn install + yarn build + # check if installation succeeded and migrate db + sudo -u www-data php bin/console partdb:check-requirements + sudo -u www-data php bin/console doctrine:migrations:migrate + sudo -u www-data php bin/console cache:clear + # we can mess with the production db because we have a very recent backup + rsync -av --exclude=$GITREPO/.git* $GITREPO/ $PRODENV-test + + echo "* The new Part-DB version can now be tested. You may need to merge .env.local with .env and check yaml files in config/." + echo "4" > $FILE + else + echo "* Invalid stage: $stage, expected 3" + exit 3 + fi + ;; + "4") + if [ "$stage" = "4" ]; then + echo "Stage 4: Put Part-DB back in production mode, retaining the old copy" + # copy all to prod environment + mv $PRODENV $PRODENV-old + mv $PRODENV-test $PRODENV + # remove link to maintenance PartDB VHost + rm $WEB_ENABLED/$LIVE_CONF $WEB_ENABLED/$TEST_CONF + # link the new partdb version + ln -sf $WEB_AVAILABLE/$LIVE_CONF $WEB_ENABLED/$LIVE_CONF + # reload apache + if ! [ `systemctl reload $WEBSERVICE && systemctl is-active --quiet $WEBSERVICE` ]; then + echo "* Webserver restart failed! Please check your $WEBSERVICE site configurations." + break + fi + echo "" + echo "*** Done. ***" + rm $FILE + else + echo "* Invalid stage: $stage, expected 4" + exit 4 + fi + ;; + esac + if [ -f $FILE ]; then + stage=$(<$FILE) + else + stage="1" + fi + else + echo "Update process aborted before stage $curstage." + break + fi +done +exit 0 From bc226f0bbb13b53c947d5f8a52bdbe1954b9d743 Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Fri, 7 Nov 2025 14:31:06 +0100 Subject: [PATCH 4/9] Update tests.yml dont stop if codecov upload fails --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 66e2f40c0..f277c7704 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -133,7 +133,7 @@ jobs: with: env_vars: PHP_VERSION,DB_TYPE token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true + fail_ci_if_error: false - name: Test app:clean-attachments run: php bin/console partdb:attachments:clean-unused -n From 1d6b63405b2e6f5551ae9b0d9544a012af5be060 Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Mon, 16 Feb 2026 09:00:49 +0100 Subject: [PATCH 5/9] Update PartSearchFilter.php --- src/DataTables/Filters/PartSearchFilter.php | 38 ++++++++++----------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index a5af37593..bae4c94eb 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -139,28 +139,23 @@ public function apply(QueryBuilder $queryBuilder): void return; } + expressions = new array(); + params = new array(); + //Use equal expression to just search for exact numeric matches if ($search_dbId) { $expressions[] = $queryBuilder->expr()->eq('part.id', ':id_exact'); - $queryBuilder->setParameter('id_exact', (int) $this->keyword, + params[] = new \Doctrine\ORM\Query\Parameter('id_exact', (int) $this->keyword, ParameterType::INTEGER); - return; } - - if($this->regex) { + else if($this->regex) { //Convert the fields to search to a list of expressions $expressions = array_map(function (string $field): string { return sprintf("REGEXP(%s, :search_query) = TRUE", $field); }, $fields_to_search); - - //Add Or concatenation of the expressions to our query - $queryBuilder->andWhere( - $queryBuilder->expr()->orX(...$expressions) - ); - + //For regex, we pass the query as is, save html special chars - $queryBuilder->setParameter('search_query', $this->keyword); - return; + params[] = new \Doctrine\ORM\Query\Parameter('search_query', $this->keyword); } else { //Escape % and _ characters in the keyword $this->keyword = str_replace(['%', '_'], ['\%', '\_'], $this->keyword); @@ -169,7 +164,7 @@ public function apply(QueryBuilder $queryBuilder): void $tokens = explode(' ', $this->keyword, 5); //Perform search of every single token in every selected field - //AND-combine the results (all tokens must be present in any result, but the order does not matter) + //AND-combine the results (all tokens must be present in any of the results) for ($i = 0; $i < sizeof($tokens); $i++) { $this->it = $i; $tokens[$i] = trim($tokens[$i]); @@ -184,15 +179,18 @@ public function apply(QueryBuilder $queryBuilder): void }, $fields_to_search); //Aggregate the parameters for consolidated commission - $params[] = new \Doctrine\ORM\Query\Parameter('search_query' . $i, '%' . $tokens[$i] . '%'); - - //Add Or concatenation of the expressions to our query - $queryBuilder->andWhere( - $queryBuilder->expr()->orX(...$expressions) - ); + $params[] = new \Doctrine\ORM\Query\Parameter('search_query' . $i, + '%' . $tokens[$i] . '%'); } - $queryBuilder->setParameters(new ArrayCollection($params)); } + + //Add Or concatenation of the expressions to our query + $queryBuilder->andWhere( + $queryBuilder->expr()->orX(...$expressions) + ); + $queryBuilder->setParameters( + new \Doctrine\Common\Collections\ArrayCollection($params) + ); } public function getKeyword(): string From 449b9446d24d10cf2b67f04c19ec84eaf9e42f5e Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Mon, 16 Feb 2026 09:03:12 +0100 Subject: [PATCH 6/9] Update PartSearchFilter.php --- src/DataTables/Filters/PartSearchFilter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index bae4c94eb..662bbf823 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -139,8 +139,8 @@ public function apply(QueryBuilder $queryBuilder): void return; } - expressions = new array(); - params = new array(); + $expressions = array(); + $params = array(); //Use equal expression to just search for exact numeric matches if ($search_dbId) { From d903c1c93105b5668ba45fa265ff79f02949d02a Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Mon, 16 Feb 2026 09:26:05 +0100 Subject: [PATCH 7/9] Update PartSearchFilter.php --- src/DataTables/Filters/PartSearchFilter.php | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index 662bbf823..7667a762d 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -139,8 +139,8 @@ public function apply(QueryBuilder $queryBuilder): void return; } - $expressions = array(); - $params = array(); + $expressions = []; + $params = []; //Use equal expression to just search for exact numeric matches if ($search_dbId) { @@ -148,7 +148,7 @@ public function apply(QueryBuilder $queryBuilder): void params[] = new \Doctrine\ORM\Query\Parameter('id_exact', (int) $this->keyword, ParameterType::INTEGER); } - else if($this->regex) { + //Convert the fields to search to a list of expressions $expressions = array_map(function (string $field): string { return sprintf("REGEXP(%s, :search_query) = TRUE", $field); @@ -184,13 +184,16 @@ public function apply(QueryBuilder $queryBuilder): void } } - //Add Or concatenation of the expressions to our query - $queryBuilder->andWhere( - $queryBuilder->expr()->orX(...$expressions) - ); $queryBuilder->setParameters( new \Doctrine\Common\Collections\ArrayCollection($params) ); + //Guard condition + if (!empty($expressions)) { + //Add Or concatenation of the expressions to our query + $queryBuilder->andWhere( + $queryBuilder->expr()->orX(...$expressions) + ); + } } public function getKeyword(): string From e5b3507bcc9116023129391de1156f138356ac6b Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Mon, 16 Feb 2026 09:28:48 +0100 Subject: [PATCH 8/9] Update PartSearchFilter.php --- src/DataTables/Filters/PartSearchFilter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index 7667a762d..faeaec958 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -148,7 +148,8 @@ public function apply(QueryBuilder $queryBuilder): void params[] = new \Doctrine\ORM\Query\Parameter('id_exact', (int) $this->keyword, ParameterType::INTEGER); } - + + if ($this->regex) { //Convert the fields to search to a list of expressions $expressions = array_map(function (string $field): string { return sprintf("REGEXP(%s, :search_query) = TRUE", $field); From a131d1cd3bc90e6bcfcde813336529c84ef53e9c Mon Sep 17 00:00:00 2001 From: d-buchmann Date: Mon, 16 Feb 2026 09:59:50 +0100 Subject: [PATCH 9/9] Update PartSearchFilter.php --- src/DataTables/Filters/PartSearchFilter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataTables/Filters/PartSearchFilter.php b/src/DataTables/Filters/PartSearchFilter.php index faeaec958..dfc0d9247 100644 --- a/src/DataTables/Filters/PartSearchFilter.php +++ b/src/DataTables/Filters/PartSearchFilter.php @@ -145,7 +145,7 @@ public function apply(QueryBuilder $queryBuilder): void //Use equal expression to just search for exact numeric matches if ($search_dbId) { $expressions[] = $queryBuilder->expr()->eq('part.id', ':id_exact'); - params[] = new \Doctrine\ORM\Query\Parameter('id_exact', (int) $this->keyword, + $params[] = new \Doctrine\ORM\Query\Parameter('id_exact', (int) $this->keyword, ParameterType::INTEGER); } @@ -156,7 +156,7 @@ public function apply(QueryBuilder $queryBuilder): void }, $fields_to_search); //For regex, we pass the query as is, save html special chars - params[] = new \Doctrine\ORM\Query\Parameter('search_query', $this->keyword); + $params[] = new \Doctrine\ORM\Query\Parameter('search_query', $this->keyword); } else { //Escape % and _ characters in the keyword $this->keyword = str_replace(['%', '_'], ['\%', '\_'], $this->keyword);