feat(helm): added cert-postgresql template to helm (#1620)

* feat(helm): added cert-postgresql template to helm

* use js-tiktoken (pure js) in favor of tiktoken (wasm)

Author: waleedlatif1

apps/sim/lib/tokenization/estimators.ts

@@ -2,7 +2,7 @@
  * Token estimation and accurate counting functions for different providers
  */
 
-import { encoding_for_model, type Tiktoken } from 'tiktoken'
+import { encodingForModel, type Tiktoken } from 'js-tiktoken'
 import { createLogger } from '@/lib/logs/console/logger'
 import { MIN_TEXT_LENGTH_FOR_ESTIMATION, TOKENIZATION_CONFIG } from '@/lib/tokenization/constants'
 import type { TokenEstimate } from '@/lib/tokenization/types'
@@ -21,12 +21,12 @@ function getEncoding(modelName: string): Tiktoken {
   }
 
   try {
-    const encoding = encoding_for_model(modelName as Parameters<typeof encoding_for_model>[0])
+    const encoding = encodingForModel(modelName as Parameters<typeof encodingForModel>[0])
     encodingCache.set(modelName, encoding)
     return encoding
   } catch (error) {
     logger.warn(`Failed to get encoding for model ${modelName}, falling back to cl100k_base`)
-    const encoding = encoding_for_model('gpt-4')
+    const encoding = encodingForModel('gpt-4')
     encodingCache.set(modelName, encoding)
     return encoding
   }
@@ -79,7 +79,7 @@ export function truncateToTokenLimit(
     }
 
     const truncatedTokens = tokens.slice(0, maxTokens)
-    const truncatedText = new TextDecoder().decode(encoding.decode(truncatedTokens))
+    const truncatedText = encoding.decode(truncatedTokens)
 
     logger.warn(
       `Truncated text from ${tokens.length} to ${maxTokens} tokens (${text.length} to ${truncatedText.length} chars)`
@@ -160,9 +160,6 @@ export function batchByTokenLimit(
  * Clean up cached encodings (call when shutting down)
  */
 export function clearEncodingCache(): void {
-  for (const encoding of encodingCache.values()) {
-    encoding.free()
-  }
   encodingCache.clear()
   logger.info('Cleared tiktoken encoding cache')
 }

apps/sim/next.config.ts

@@ -75,7 +75,7 @@ const nextConfig: NextConfig = {
   turbopack: {
     resolveExtensions: ['.tsx', '.ts', '.jsx', '.js', '.mjs', '.json'],
   },
-  serverExternalPackages: ['pdf-parse', 'tiktoken'],
+  serverExternalPackages: ['pdf-parse'],
   experimental: {
     optimizeCss: true,
     turbopackSourceMaps: false,

apps/sim/package.json

@@ -82,6 +82,7 @@
     "input-otp": "^1.4.2",
     "ioredis": "^5.6.0",
     "jose": "6.0.11",
+    "js-tiktoken": "1.0.21",
     "js-yaml": "4.1.0",
     "jwt-decode": "^4.0.0",
     "lucide-react": "^0.479.0",
@@ -119,6 +120,7 @@
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^6.6.3",
+    "@trigger.dev/build": "4.0.4",
     "@types/html-to-text": "9.0.4",
     "@types/js-yaml": "4.0.9",
     "@types/jsdom": "21.1.7",

apps/sim/trigger.config.ts

@@ -1,3 +1,4 @@
+import { additionalPackages } from '@trigger.dev/build/extensions/core'
 import { defineConfig } from '@trigger.dev/sdk'
 import { env } from './lib/env'
 
@@ -13,4 +14,12 @@ export default defineConfig({
     },
   },
   dirs: ['./background'],
+  build: {
+    extensions: [
+      // pdf-parse has native bindings, keep as external package
+      additionalPackages({
+        packages: ['pdf-parse'],
+      }),
+    ],
+  },
 })

bun.lock

@@ -0,0 +1,35 @@
+{{- if and .Values.postgresql.enabled .Values.postgresql.tls.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "sim.fullname" . }}-postgresql-tls-certificate
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "sim.postgresql.labels" . | nindent 4 }}
+spec:
+  secretName: {{ .Values.postgresql.tls.certificatesSecret }}
+  duration: {{ .Values.postgresql.tls.duration | default "87600h" }}  # Default: 10 years
+  renewBefore: {{ .Values.postgresql.tls.renewBefore | default "2160h" }}  # Default: 90 days before expiry
+  isCA: false
+  {{- if .Values.postgresql.tls.rotationPolicy }}
+  rotationPolicy: {{ .Values.postgresql.tls.rotationPolicy }}
+  {{- end }}
+  privateKey:
+    algorithm: {{ .Values.postgresql.tls.privateKey.algorithm | default "RSA" }}
+    size: {{ .Values.postgresql.tls.privateKey.size | default 4096 }}
+  usages:
+    - server auth
+    - client auth
+  dnsNames:
+  - {{ include "sim.fullname" . }}-postgresql
+  - {{ include "sim.fullname" . }}-postgresql.{{ .Release.Namespace }}.svc.cluster.local
+  {{- with .Values.postgresql.tls.additionalDnsNames }}
+  {{- toYaml . | nindent 2 }}
+  {{- end }}
+  issuerRef:
+    name: {{ .Values.postgresql.tls.issuerRef.name }}
+    kind: {{ .Values.postgresql.tls.issuerRef.kind | default "ClusterIssuer" }}
+    {{- if .Values.postgresql.tls.issuerRef.group }}
+    group: {{ .Values.postgresql.tls.issuerRef.group }}
+    {{- end }}
+{{- end }}

helm/sim/templates/certificate-postgresql.yaml

@@ -290,9 +290,28 @@ postgresql:
       - ReadWriteOnce
 
   # SSL/TLS configuration (enable for production deployments with certificates)
+  # Requires cert-manager to be installed in the cluster
   tls:
     enabled: false
     certificatesSecret: postgres-tls-secret
+    # Certificate configuration (only used if enabled)
+    duration: "87600h"  # 10 years (default)
+    renewBefore: "2160h"  # Renew 90 days before expiry (default)
+    rotationPolicy: ""  # Set to "Always" to rotate private key on renewal (recommended for security)
+    privateKey:
+      algorithm: RSA  # RSA or ECDSA
+      size: 4096  # Key size in bits
+    # Issuer reference (REQUIRED if tls.enabled is true)
+    issuerRef:
+      name: selfsigned-cluster-issuer  # Name of your cert-manager Issuer/ClusterIssuer
+      kind: ClusterIssuer  # ClusterIssuer or Issuer
+      group: ""  # Optional: cert-manager.io (leave empty for default)
+    # Additional DNS names (optional)
+    additionalDnsNames: []
+    # Example:
+    # additionalDnsNames:
+    #   - postgres.example.com
+    #   - db.example.com
 
   # PostgreSQL configuration
   config:

helm/sim/values.yaml

@@ -42,7 +42,6 @@
     "postgres": "^3.4.5",
     "remark-gfm": "4.0.1",
     "socket.io-client": "4.8.1",
-    "tiktoken": "1.0.22",
     "twilio": "5.9.0"
   },
   "devDependencies": {