diff --git a/n8n/n8n-queue-stack.yaml b/n8n/n8n-queue-stack.yaml new file mode 100644 index 0000000..52de906 --- /dev/null +++ b/n8n/n8n-queue-stack.yaml @@ -0,0 +1,266 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: n8n +--- +# 1. THE ISOLATED CNPG POSTGRES CLUSTER FOR N8N ONLY +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: n8n-db + namespace: n8n +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:16.4 + storage: + size: 15Gi + storageClass: longhorn + bootstrap: + initdb: + database: n8n_prod + owner: n8n_admin +--- +# 2. THE SIMPLE DEDICATED VALKEY CACHE +apiVersion: apps/v1 +kind: Deployment +metadata: + name: n8n-valkey + namespace: n8n +spec: + replicas: 1 + selector: + matchLabels: + app: n8n-valkey + template: + metadata: + labels: + app: n8n-valkey + spec: + containers: + - name: valkey + image: valkey/valkey:8.0-alpine + args: ["--maxmemory", "256mb", "--maxmemory-policy", "allkeys-lru"] + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: n8n-valkey-svc + namespace: n8n +spec: + ports: + - port: 6379 + selector: + app: n8n-valkey +--- +# 3. APPLICATION STORAGE FOR LOCAL BINARY DATA STREAMING +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: n8n-main-pvc + namespace: n8n +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi +--- +# 4. N8N MAIN ORCHESTRATOR & WEB UI DEPLOYMENT +apiVersion: apps/v1 +kind: Deployment +metadata: + name: n8n-main + namespace: n8n +spec: + replicas: 1 + selector: + matchLabels: + app: n8n-main + template: + metadata: + labels: + app: n8n-main + spec: + # 👇 ADD THIS BLOCK HERE TO FIX SCENARIO A 👇 + securityContext: + fsGroup: 1000 + runAsUser: 1000 + containers: + - name: n8n + image: docker.io/n8nio/n8n:1.64.3 + command: ["n8n"] + ports: + - containerPort: 5678 + env: + - name: N8N_HOST + value: "n8n.needlestack.boo" + - name: N8N_PORT + value: "5678" + - name: N8N_PROTOCOL + value: "https" + - name: NODE_ENV + value: "production" + - name: WEBHOOK_URL + value: "https://n8n.needlestack.boo" + # Queue Architecture Settings + - name: EXECUTIONS_MODE + value: "queue" + - name: QUEUE_BULL_REDIS_HOST + value: "n8n-valkey-svc" + - name: QUEUE_BULL_REDIS_PORT + value: "6379" + - name: N8N_ENCRYPTION_KEY + value: "68ffe0f5c9f0424b277b1930fec9ae31" # ⚠️ REPLACE THIS WITH YOUR GENERATED KEY + # Database Configuration (CNPG Integration) + - name: DB_TYPE + value: "postgresdb" + - name: DB_POSTGRESDB_HOST + value: "n8n-db-rw" + - name: DB_POSTGRESDB_PORT + value: "5432" + - name: DB_POSTGRESDB_DATABASE + value: "n8n_prod" + - name: DB_POSTGRESDB_USER + value: "n8n_admin" + - name: DB_POSTGRESDB_PASSWORD + valueFrom: + secretKeyRef: + name: n8n-db-app + key: password + # Database SSL Handshake Overrides + - name: DB_POSTGRESDB_SSL + value: "true" + - name: DB_POSTGRESDB_SSL_REJECT_UNAUTHORIZED + value: "false" + # Resource & Ingestion Optimization Settings + - name: N8N_ENFORCE_SETTINGS_FILE_DATA_PROPERTY + value: "true" + - name: EXECUTIONS_DATA_PRUNE + value: "true" + - name: EXECUTIONS_DATA_MAX_AGE + value: "48" + resources: + requests: + memory: "250Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "1000m" + volumeMounts: + - name: n8n-storage + mountPath: /home/node/.n8n + volumes: + - name: n8n-storage + persistentVolumeClaim: + claimName: n8n-main-pvc +--- +# 5. N8N WORKER DEPLOYMENT (THE EXECUTION ENGINE) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: n8n-worker + namespace: n8n +spec: + replicas: 1 + selector: + matchLabels: + app: n8n-worker + template: + metadata: + labels: + app: n8n-worker + spec: + containers: + - name: worker + image: docker.io/n8nio/n8n:1.64.3 + command: ["n8n", "worker"] + env: + - name: NODE_ENV + value: "production" + # Queue Architecture Settings + - name: EXECUTIONS_MODE + value: "queue" + - name: QUEUE_BULL_REDIS_HOST + value: "n8n-valkey-svc" + - name: QUEUE_BULL_REDIS_PORT + value: "6379" + - name: N8N_ENCRYPTION_KEY + value: "68ffe0f5c9f0424b277b1930fec9ae31" # ⚠️ MUST MATCH THE MAIN KEY EXACTLY ABOVE + # Database Configuration (CNPG Integration) + - name: DB_TYPE + value: "postgresdb" + - name: DB_POSTGRESDB_HOST + value: "n8n-db-rw" + - name: DB_POSTGRESDB_PORT + value: "5432" + - name: DB_POSTGRESDB_DATABASE + value: "n8n_prod" + - name: DB_POSTGRESDB_USER + value: "n8n_admin" + - name: DB_POSTGRESDB_PASSWORD + valueFrom: + secretKeyRef: + name: n8n-db-app + key: password + # Database SSL Handshake Overrides + - name: DB_POSTGRESDB_SSL + value: "true" + - name: DB_POSTGRESDB_SSL_REJECT_UNAUTHORIZED + value: "false" + # Resource & Ingestion Optimization Settings + - name: N8N_ENFORCE_SETTINGS_FILE_DATA_PROPERTY + value: "true" + resources: + requests: + memory: "250Mi" + cpu: "200m" + limits: + memory: "1536Mi" + cpu: "1500m" +--- +# 6. N8N NETWORK SERVICE MESH +apiVersion: v1 +kind: Service +metadata: + name: n8n-svc + namespace: n8n +spec: + ports: + - port: 5678 + targetPort: 5678 + selector: + app: n8n-main +--- +# 7. THE INGRESS ROUTING WITH WEBSOCKET SUPPORT +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: n8n-ingress + namespace: n8n + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-production" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/proxy-body-size: "100m" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +spec: + ingressClassName: nginx + tls: + - secretName: n8n-tls-secret + hosts: + - n8n.needlestack.boo + rules: + - host: n8n.needlestack.boo + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: n8n-svc + port: + number: 5678 diff --git a/paperless-ngx/paperless-stack.yaml b/paperless-ngx/paperless-stack.yaml index 6ed1f8d..5a54b30 100644 --- a/paperless-ngx/paperless-stack.yaml +++ b/paperless-ngx/paperless-stack.yaml @@ -125,6 +125,13 @@ spec: value: "10.42.0.0/16,10.43.0.0/16" # Trusts the internal k3s pod/service CIDR blocks - name: PAPERLESS_TIME_ZONE value: "Europe/London" + # 👇 ADD THESE THREE TO ENABLE OFFICE INGESTION 👇 + - name: PAPERLESS_TIKA_ENABLED + value: "true" + - name: PAPERLESS_TIKA_ENDPOINT + value: "http://paperless-tika-svc:9998" + - name: PAPERLESS_TIKA_GOTENBERG_ENDPOINT + value: "http://paperless-gotenberg-svc:3000" volumeMounts: - name: paperless-storage mountPath: /usr/src/paperless/data @@ -156,6 +163,9 @@ metadata: # 👇 ADD THESE THREE FOR CSRF / PROXY ALIGNMENT 👇 nginx.ingress.kubernetes.io/proxy-body-size: "50m" # Essential later for uploading large PDFs! nginx.ingress.kubernetes.io/ssl-redirect: "true" # 👈 This handles forcing HTTPS routing safely + # 👇 ADD THESE TWO WEBSOCKET TIMEOUT/UPGRADE ANNOTATIONS 👇 + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" spec: ingressClassName: nginx tls: @@ -173,3 +183,73 @@ spec: name: paperless-svc port: number: 8000 +--- +# 6. GOTENBERG (OFFICE CONVERSION ENGINE) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: paperless-gotenberg + namespace: paperless +spec: + replicas: 1 + selector: + matchLabels: + app: paperless-gotenberg + template: + metadata: + labels: + app: paperless-gotenberg + spec: + containers: + - name: gotenberg + image: gotenberg/gotenberg:8.15.0 # Check for latest tag if preferred + ports: + - containerPort: 3000 + # 👇 ADD THIS ENV BLOCK TO EXTEND TIMEOUT TO 120 SECONDS 👇 + env: + - name: GOTENBERG_API_TIMEOUT + value: "120s" +--- +apiVersion: v1 +kind: Service +metadata: + name: paperless-gotenberg-svc + namespace: paperless +spec: + ports: + - port: 3000 + selector: + app: paperless-gotenberg +--- +# 7. APACHE TIKA (METADATA TEXT EXTRACTION) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: paperless-tika + namespace: paperless +spec: + replicas: 1 + selector: + matchLabels: + app: paperless-tika + template: + metadata: + labels: + app: paperless-tika + spec: + containers: + - name: tika + image: apache/tika:3.0.0.0-full # The "-full" variant contains the proper bindings + ports: + - containerPort: 9998 +--- +apiVersion: v1 +kind: Service +metadata: + name: paperless-tika-svc + namespace: paperless +spec: + ports: + - port: 9998 + selector: + app: paperless-tika