Confluence synchrony pod is not starting.

[mrkhachaturov]

Trying to install confluence via helm in v1.29.5+k3s1
Chart version 1.20.0
App version 8.5.9

Using default values, only enabled synchrony.

USER-SUPPLIED VALUES:
additionalConfigMaps: []
additionalContainers: []
additionalFiles: []
additionalHosts: []
additionalInitContainers: []
additionalLabels: {}
affinity: {}
atlassianAnalyticsAndSupport:
  analytics:
    enabled: true
  helmValues:
    enabled: true
confluence:
  accessLog:
    enabled: true
    localHomeSubPath: logs
    mountPath: /opt/atlassian/confluence/logs
  additionalBundledPlugins: []
  additionalCertificates:
    customCmd: null
    secretName: null
  additionalEnvironmentVariables: []
  additionalJvmArgs: []
  additionalLibraries: []
  additionalPorts: []
  additionalVolumeClaimTemplates: []
  additionalVolumeMounts: []
  clustering:
    enabled: false
    usePodNameAsClusterNodeName: true
  containerSecurityContext: {}
  forceConfigUpdate: false
  hazelcastService:
    annotations: {}
    enabled: false
    port: 5701
    type: ClusterIP
  jvmDebug:
    enabled: false
  license:
    secretKey: license-key
    secretName: null
  livenessProbe:
    customProbe: {}
    enabled: false
    failureThreshold: 12
    initialDelaySeconds: 60
    periodSeconds: 5
    timeoutSeconds: 1
  ports:
    hazelcast: 5701
    http: 8090
  postStart:
    command: null
  readinessProbe:
    customProbe: {}
    enabled: true
    failureThreshold: 6
    initialDelaySeconds: 10
    periodSeconds: 5
    timeoutSeconds: 1
  resources:
    container:
      requests:
        cpu: "2"
        memory: 2G
    jvm:
      maxHeap: 1g
      minHeap: 1g
      reservedCodeCache: 256m
  s3AttachmentsStorage:
    bucketName: null
    bucketRegion: null
    endpointOverride: null
  securityContext:
    fsGroup: 2002
  securityContextEnabled: true
  seraphConfig:
    autoLoginCookieAge: "1209600"
    generateByHelm: false
  service:
    annotations: {}
    contextPath: null
    loadBalancerIP: null
    port: 80
    sessionAffinity: None
    sessionAffinityConfig:
      clientIP:
        timeoutSeconds: null
    type: ClusterIP
  setPermissions: true
  shutdown:
    command: /shutdown-wait.sh
    terminationGracePeriodSeconds: 25
  startupProbe:
    enabled: false
    failureThreshold: 120
    initialDelaySeconds: 60
    periodSeconds: 5
  tomcatConfig:
    acceptCount: "10"
    connectionTimeout: "20000"
    customServerXml: ""
    debug: "0"
    enableLookups: "false"
    generateByHelm: false
    maxHttpHeaderSize: "8192"
    maxThreads: "100"
    mgmtPort: "8000"
    minSpareThreads: "10"
    port: "8090"
    protocol: org.apache.coyote.http11.Http11NioProtocol
    proxyInternalIps: null
    proxyName: null
    proxyPort: null
    redirectPort: "8443"
    scheme: null
    secure: null
    trustedProxies: null
    uriEncoding: UTF-8
  topologySpreadConstraints: []
  umask: "0022"
  useHelmReleaseNameAsContainerName: false
database:
  credentials:
    passwordSecretKey: password
    secretName: null
    usernameSecretKey: username
  type: null
  url: null
fluentd:
  command: null
  customConfigFile: false
  elasticsearch:
    enabled: true
    hostname: elasticsearch
    indexNamePrefix: confluence
  enabled: false
  extraVolumes: []
  fluentdCustomConfig: {}
  httpPort: 9880
  imageRepo: fluent/fluentd-kubernetes-daemonset
  imageTag: v1.11.5-debian-elasticsearch7-1.2
  resources: {}
image:
  pullPolicy: IfNotPresent
  repository: atlassian/confluence
  tag: ""
ingress:
  annotations: {}
  className: nginx
  create: false
  host: null
  https: true
  maxBodySize: 250m
  nginx: true
  openShiftRoute: false
  path: null
  proxyConnectTimeout: 60
  proxyReadTimeout: 60
  proxySendTimeout: 60
  routeHttpHeaders: {}
  tlsSecretName: null
monitoring:
  exposeJmxMetrics: false
  fetchJmxExporterJar: true
  grafana:
    createDashboards: false
    dashboardAnnotations: {}
    dashboardLabels: {}
  jmxExporterCustomConfig: {}
  jmxExporterCustomJarLocation: null
  jmxExporterImageRepo: bitnami/jmx-exporter
  jmxExporterImageTag: 0.18.0
  jmxExporterInitContainer:
    customSecurityContext: {}
    resources: {}
    runAsRoot: true
  jmxExporterPort: 9999
  jmxExporterPortType: ClusterIP
  jmxServiceAnnotations: {}
  serviceMonitor:
    create: false
    prometheusLabelSelector: {}
    scrapeIntervalSeconds: 30
nodeSelector: {}
opensearch:
  credentials:
    createSecret: true
    existingSecretRef:
      name: null
  enabled: false
  envFrom:
  - secretRef:
      name: opensearch-initial-password
  extraEnvs:
  - name: plugins.security.ssl.http.enabled
    value: "false"
  persistence:
    size: 10Gi
  resources:
    requests:
      cpu: 1
      memory: 1Gi
  singleNode: true
openshift:
  runWithRestrictedSCC: false
ordinals:
  enabled: false
  start: 0
podAnnotations: {}
podDisruptionBudget:
  annotations: {}
  enabled: false
  labels: {}
  maxUnavailable: null
  minAvailable: null
podLabels: {}
priorityClassName: null
replicaCount: 1
schedulerName: null
serviceAccount:
  annotations: {}
  clusterRole:
    create: false
    name: null
  clusterRoleBinding:
    create: false
    name: null
  create: true
  eksIrsa:
    roleArn: null
  imagePullSecrets: []
  name: null
  role:
    create: true
  roleBinding:
    create: true
synchrony:
  additionalCertificates:
    customCmd: null
    secretName: null
  additionalJvmArgs: []
  additionalLibraries: []
  additionalPorts: []
  additionalVolumeMounts: []
  containerSecurityContext: {}
  enabled: true
  podAnnotations: {}
  ports:
    hazelcast: 5701
    http: 8091
  readinessProbe:
    failureThreshold: 10
    healthcheckPath: /synchrony/heartbeat
    initialDelaySeconds: 5
    periodSeconds: 1
  replicaCount: 1
  resources:
    container:
      requests:
        cpu: "2"
        memory: 2.5G
    jvm:
      maxHeap: 2g
      minHeap: 1g
      stackSize: 2048k
  securityContext:
    fsGroup: 2002
  securityContextEnabled: true
  service:
    annotations: {}
    loadBalancerIP: null
    port: 80
    type: ClusterIP
  setPermissions: true
  shutdown:
    terminationGracePeriodSeconds: 25
  topologySpreadConstraints: []
testPods:
  affinity: {}
  annotations: {}
  image:
    permissionsTestContainer: debian:stable-slim
    statusTestContainer: alpine:latest
  labels: {}
  nodeSelector: {}
  resources: {}
  schedulerName: null
  tolerations: []
tolerations: []
volumes:
  additional: []
  additionalSynchrony: []
  defaultPermissionsMode: 484
  localHome:
    customVolume: {}
    mountPath: /var/atlassian/application-data/confluence
    persistentVolumeClaim:
      create: false
      resources:
        requests:
          storage: 1Gi
      storageClassName: null
    persistentVolumeClaimRetentionPolicy:
      whenDeleted: null
      whenScaled: null
  sharedHome:
    customVolume: {}
    mountPath: /var/atlassian/application-data/shared-home
    nfsPermissionFixer:
      command: null
      enabled: true
      imageRepo: alpine
      imageTag: latest
      mountPath: /shared-home
      resources: {}
    persistentVolumeClaim:
      create: false
      resources:
        requests:
          storage: 1Gi
      storageClassName: null
    subPath: null
  synchronyHome:
    customVolume: {}
    mountPath: /var/atlassian/application-data/confluence
    persistentVolumeClaim:
      create: true
      resources:
        requests:
          storage: 1Gi
      storageClassName: null
    persistentVolumeClaimRetentionPolicy:
      whenDeleted: null
      whenScaled: nul

Running with helm install confluence atlassian-data-center/confluence -n atlassian -f default-values.yaml

Synchrony pods logs:

WARNING: update-vals already refers to: #'clojure.core/update-vals in namespace: clojure.tools.analyzer, being replaced by: #'clojure.tools.analyzer.utils/update-vals
WARNING: update-keys already refers to: #'clojure.core/update-keys in namespace: clojure.tools.analyzer, being replaced by: #'clojure.tools.analyzer.utils/update-keys
WARNING: update-vals already refers to: #'clojure.core/update-vals in namespace: clojure.tools.analyzer.passes, being replaced by: #'clojure.tools.analyzer.utils/update-vals
WARNING: update-vals already refers to: #'clojure.core/update-vals in namespace: clojure.tools.analyzer.passes.uniquify, being replaced by: #'clojure.tools.analyzer.utils/update-vals
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/opt/atlassian/confluence/confluence/WEB-INF/packages/synchrony-standalone.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/opt/atlassian/confluence/confluence/WEB-INF/lib/org.apache.logging.log4j_log4j-slf4j-impl-2.19.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Reload4jLoggerFactory]
Starting synchrony
2024-06-16 15:18:37,169 ERROR [main] [synchrony.logging] uncaught exception {:throwable #error {
	 :cause "Cannot invoke \"java.lang.CharSequence.subSequence(int, int)\" because \"text\" is null"
	 :via
	 [{:type clojure.lang.ExceptionInfo
	   :message "error while starting state #'synchrony.querydsl.db/pool-datasource-st"
	   :data {:var #'synchrony.querydsl.db/pool-datasource-st}
	   :at [mount.lite$start$fn__24145 invoke "lite.clj" 153]}
	  {:type java.lang.NullPointerException
	   :message "Cannot invoke \"java.lang.CharSequence.subSequence(int, int)\" because \"text\" is null"
	   :at [instaparse.gll$sub_sequence invokeStatic "gll.clj" 216]}]
	 	 [[instaparse.gll$sub_sequence invokeStatic "gll.clj" 216]
	  [instaparse.gll$sub_sequence invoke "gll.clj" 211]
	  [instaparse.gll$string_parse invokeStatic "gll.clj" 623]
	  [instaparse.gll$string_parse invoke "gll.clj" 618]
	  [instaparse.gll$_parse invokeStatic "gll.clj" 142]
	  [instaparse.gll$_parse invoke "gll.clj" 134]
	  [instaparse.gll$push_listener$fn__19859 invoke "gll.clj" 387]
	  [instaparse.gll$step invokeStatic "gll.clj" 449]
	  [instaparse.gll$step invoke "gll.clj" 443]
	  [instaparse.gll$run invokeStatic "gll.clj" 467]
	  [instaparse.gll$run invoke "gll.clj" 451]
	  [instaparse.gll$run invokeStatic "gll.clj" 453]
	  [instaparse.gll$run invoke "gll.clj" 451]
	  [instaparse.gll$parse invokeStatic "gll.clj" 983]
	  [instaparse.gll$parse invoke "gll.clj" 978]
	  [instaparse.core$parse invokeStatic "core.clj" 94]
	  [instaparse.core$parse doInvoke "core.clj" 47]
	  [clojure.lang.RestFn invoke "RestFn.java" 428]
	  [instaparse.core.Parser invoke "core.clj" 141]
	  [ginga.jdbc$parse_uri invokeStatic "jdbc.clj" 40]
	  [ginga.jdbc$parse_uri invoke "jdbc.clj" 39]
	  [synchrony.querydsl.db$make_pool_db_spec invokeStatic "db.clj" 83]
	  [synchrony.querydsl.db$make_pool_db_spec invoke "db.clj" 82]
	  [synchrony.querydsl.db$fn__25444$fn__25447 invoke "db.clj" 111]
	  [mount.lite.State start_STAR_ "lite.clj" 44]
	  [mount.lite$start$fn__24145 invoke "lite.clj" 151]
	  [mount.lite$start invokeStatic "lite.clj" 150]
	  [mount.lite$start invoke "lite.clj" 136]
	  [mount.lite$start invokeStatic "lite.clj" 141]
	  [mount.lite$start invoke "lite.clj" 136]
	  [synchrony.core$start_BANG_ invokeStatic "core.clj" 21]
	  [synchrony.core$start_BANG_ invoke "core.clj" 21]
	  [synchrony.core$_main invokeStatic "core.clj" 42]
	  [synchrony.core$_main doInvoke "core.clj" 40]
	  [clojure.lang.RestFn applyTo "RestFn.java" 140]
	  [synchrony.core main nil -1]]}}

[mrkhachaturov]

Same if pvs is defined

volumes:
  localHome:
    persistentVolumeClaim:
      create: true
      storageClassName: longhorn-appdata

  sharedHome:
    persistentVolumeClaim:
      create: true
      storageClassName: longhorn-shared

  synchronyHome:
    persistentVolumeClaim:
      create: true
      storageClassName: longhorn-appdata

      # -- Specifies the standard K8s resource requests for the synchrony-home
      # volume claims.
      #
      resources:
        requests:
          storage: 1Gi

Config map for syncrhony is generaed:

#!/usr/bin/env bash

java \
   -Xms1g \
   -Xmx2g \
   -Xss2048k \
   -Dsynchrony.port=8091 \
   -Dcluster.listen.port=5701 \
   -XX:ActiveProcessorCount=2 \
   -classpath /opt/atlassian/confluence/confluence/WEB-INF/packages/synchrony-standalone.jar:/opt/atlassian/confluence/confluence/WEB-INF/lib/* \
   synchrony.core \
   sql

Pod is crashing:

Readiness probe failed: Get "http://10.42.8.124:8091/synchrony/heartbeat": dial tcp 10.42.8.124:8091: connect: connection refused

[mrkhachaturov]

Database connection config must be defined in values and also disabling readinessProbe helped me for first cold start.

readinessProbe:
              enabled: false