NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(nci-11): add Kubernetes manifests for nowchess-analysis service (#392)

Browse Source 
Co-authored-by: Janis Eccarius <eccariusjanis@gmail.com>
Reviewed-on: #392
This commit was merged in pull request #392.
This commit is contained in:
Janis
2026-06-16 19:42:04 +02:00
parent 0442352b3e
commit 63e9d0512c
66 changed files with 6523 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eu-central-1/argo-apps/spark-analytics/spark-analytics.yaml
Executable
+18
View File
@@ -0,0 +1,18 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: spark-analytics
namespace: argocd
spec:
project: default
destination:
namespace: nowchess
server: https://kubernetes.default.svc
sources:
- repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
path: ./spark-analytics/eu-central-1-prod
targetRevision: main
syncPolicy:
automated:
prune: true
selfHeal: true
eu-central-1/argo-apps/spark-cluster/spark-cluster.yaml
Executable
+18
View File
@@ -0,0 +1,18 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: spark-cluster
namespace: argocd
spec:
project: default
destination:
namespace: nowchess
server: https://kubernetes.default.svc
sources:
- repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
path: ./spark-cluster/eu-central-1-prod
targetRevision: main
syncPolicy:
automated:
prune: true
selfHeal: true
nowchess/eu-central-1-prod/kustomization.yaml
+2
View File
@@ -32,6 +32,8 @@ patches:
kind: ConfigMap
name: nowchess-frontend-env-config
images:
- name: ghcr.io/now-chess/now-chess-systems/analysis
newTag: 0.1.0
- name: ghcr.io/now-chess/now-chess-systems/account
newTag: 0.24.0
- name: ghcr.io/now-chess/now-chess-systems/analysis
nowchess/eu-central-1-staging/kustomization.yaml
+2
View File
@@ -94,6 +94,8 @@ patches:
spec:
replicas: 1
images:
- name: ghcr.io/now-chess/now-chess-systems/analysis
newTag: 0.1.0
- name: ghcr.io/now-chess/now-chess-systems/account
newTag: 0.24.0
- name: ghcr.io/now-chess/now-chess-systems/analysis
nowchess/htwg-1-prod/kustomization.yaml
+2
View File
@@ -31,6 +31,8 @@ patches:
kind: ConfigMap
name: nowchess-frontend-env-config
images:
- name: ghcr.io/now-chess/now-chess-systems/analysis
newTag: 0.1.0-x86_v2
- name: ghcr.io/now-chess/now-chess-systems/account
newTag: 0.24.0-x86_v2
- name: ghcr.io/now-chess/now-chess-systems/analysis
spark-analytics/base/cronjobs.yaml
Executable
+321
View File
@@ -0,0 +1,321 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-opening-book
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 2 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar
- /spark-output/opening-book
- "10"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-stats
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 3 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar
- /spark-output/player-stats
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-clustering
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 4 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar
- /spark-output/player-clusters
- "4"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-graph
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 5 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar
- /spark-output/player-graph
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
spark-analytics/base/kustomization.yaml
Executable
+9
View File
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- serviceaccount.yaml
- rbac.yaml
- pvc.yaml
- cronjobs.yaml
- streaming.yaml
spark-analytics/base/pvc.yaml
Executable
+27
View File
@@ -0,0 +1,27 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: spark-analytics-output
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: spark-streaming-checkpoint
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
spark-analytics/base/rbac.yaml
Executable
+32
View File
@@ -0,0 +1,32 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: spark-analytics-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create", "get", "list", "watch", "delete", "patch"]
- apiGroups: [""]
resources: ["services"]
verbs: ["create", "get", "delete"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create", "get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: spark-analytics-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
subjects:
- kind: ServiceAccount
name: spark-analytics
roleRef:
kind: Role
name: spark-analytics-driver
apiGroup: rbac.authorization.k8s.io
spark-analytics/base/serviceaccount.yaml
Executable
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: spark-analytics
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spark-analytics/base/streaming.yaml
Executable
+78
View File
@@ -0,0 +1,78 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spark-live-dashboard
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
replicas: 1
selector:
matchLabels:
app: spark-live-dashboard
template:
metadata:
labels:
app: spark-live-dashboard
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=1
- --conf
- spark.executor.memory=512m
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.LiveDashboardJob
- local:///app/analytics.jar
- /spark-checkpoint
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-checkpoint
mountPath: /spark-checkpoint
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-checkpoint
persistentVolumeClaim:
claimName: spark-streaming-checkpoint
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
spark-analytics/eu-central-1-prod/kustomization.yaml
Executable
+176
View File
@@ -0,0 +1,176 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: nowchess
resources:
- ../base
images:
- name: ghcr.io/now-chess/now-chess-systems/analytics
newTag: 0.1.0
patches:
# Pin the analytics image tag in each CronJob and the Deployment.
# The base uses :latest; overlays pin to a released tag.
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-opening-book
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar
- /spark-output/opening-book
- "10"
target:
kind: CronJob
name: spark-opening-book
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-stats
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar
- /spark-output/player-stats
target:
kind: CronJob
name: spark-player-stats
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-clustering
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar
- /spark-output/player-clusters
- "4"
target:
kind: CronJob
name: spark-player-clustering
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-graph
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar
- /spark-output/player-graph
target:
kind: CronJob
name: spark-player-graph
- patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: spark-live-dashboard
spec:
template:
spec:
containers:
- name: spark-driver
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=1
- --conf
- spark.executor.memory=512m
- --conf
- spark.driver.memory=512m
- --conf
- spark.eventLog.enabled=true
- --conf
- spark.eventLog.dir=/spark-events
- --class
- de.nowchess.analytics.LiveDashboardJob
- local:///app/analytics.jar
- /spark-checkpoint
target:
kind: Deployment
name: spark-live-dashboard
spark-cluster/base/charts/spark-9.2.10/spark/.helmignore
Executable
+25
View File
@@ -0,0 +1,25 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
# img folder
img/
# Changelog
CHANGELOG.md
spark-cluster/base/charts/spark-9.2.10/spark/Chart.lock
Executable
+6
View File
@@ -0,0 +1,6 @@
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
version: 2.22.0
digest: sha256:a8fb2fc887ead658a89598a48acde5324196fbc0509503a3eaed50a710fbfe74
generated: "2024-08-12T12:55:49.00370715Z"
spark-cluster/base/charts/spark-9.2.10/spark/Chart.yaml
Executable
+29
View File
@@ -0,0 +1,29 @@
annotations:
category: Infrastructure
images: |
- name: spark
image: docker.io/bitnami/spark:3.5.2-debian-12-r1
licenses: Apache-2.0
apiVersion: v2
appVersion: 3.5.2
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
tags:
- bitnami-common
version: 2.x.x
description: Apache Spark is a high-performance engine for large-scale computing tasks,
such as data processing, machine learning and real-time data streaming. It includes
APIs for Java, Python, Scala and R.
home: https://bitnami.com
icon: https://bitnami.com/assets/stacks/spark/img/spark-stack-220x234.png
keywords:
- apache
- spark
maintainers:
- name: Broadcom, Inc. All Rights Reserved.
url: https://github.com/bitnami/charts
name: spark
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/spark
version: 9.2.10
spark-cluster/base/charts/spark-9.2.10/spark/README.md
Executable
+547
View File
@@ -0,0 +1,547 @@
<!--- app-name: Apache Spark -->
# Bitnami package for Apache Spark
Apache Spark is a high-performance engine for large-scale computing tasks, such as data processing, machine learning and real-time data streaming. It includes APIs for Java, Python, Scala and R.
[Overview of Apache Spark](https://spark.apache.org/)
Trademarks: This software listing is packaged by Bitnami. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement.
## TL;DR
```console
helm install my-release oci://registry-1.docker.io/bitnamicharts/spark
```
Looking to use Apache Spark in production? Try [VMware Tanzu Application Catalog](https://bitnami.com/enterprise), the commercial edition of the Bitnami catalog.
## Introduction
This chart bootstraps an [Apache Spark](https://github.com/bitnami/containers/tree/main/bitnami/spark) deployment on a [Kubernetes](https://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
Apache Spark includes APIs for Java, Python, Scala and R.
Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters.
## Prerequisites
- Kubernetes 1.23+
- Helm 3.8.0+
## Installing the Chart
To install the chart with the release name `my-release`:
```console
helm install my-release oci://REGISTRY_NAME/REPOSITORY_NAME/spark
```
> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`.
These commands deploy Apache Spark on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation.
> **Tip**: List all releases using `helm list`
## Configuration and installation details
### Resource requests and limits
Bitnami charts allow setting resource requests and limits for all containers inside the chart deployment. These are inside the `resources` value (check parameter table). Setting requests is essential for production workloads and these should be adapted to your specific use case.
To make this process easier, the chart contains the `resourcesPreset` values, which automatically sets the `resources` section according to different presets. Check these presets in [the bitnami/common chart](https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15). However, in production workloads using `resourcePreset` is discouraged as it may not fully adapt to your specific needs. Find more information on container resource management in the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/).
### [Rolling vs Immutable tags](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html)
It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image.
Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist.
### Define custom configuration
To use a custom configuration, a ConfigMap should be created with the `spark-env.sh` file inside the ConfigMap. The ConfigMap name must be provided at deployment time.
To set the configuration on the master use `master.configurationConfigMap=configMapName`. To set the configuration on the worker, use `worker.configurationConfigMap=configMapName`.
These values can be set at the same time in a single ConfigMap or using two ConfigMaps. An additional `spark-defaults.conf` file can be provided in the ConfigMap. You can use both files or one without the other.
### Submit an application
To submit an application to the Apache Spark cluster, use the `spark-submit` script, which is available at [https://github.com/apache/spark/tree/master/bin](https://github.com/apache/spark/tree/master/bin).
The command below illustrates the process of deploying one of the sample applications included with Apache Spark. Replace the `k8s-apiserver-host`, `k8s-apiserver-port`, `spark-master-svc`, and `spark-master-port` placeholders with the correct master host/IP address and port for your deployment.
```console
$ ./bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--conf spark.kubernetes.container.image=bitnami/spark:3 \
--master k8s://https://k8s-apiserver-host:k8s-apiserver-port \
--conf spark.kubernetes.driverEnv.SPARK_MASTER_URL=spark://spark-master-svc:spark-master-port \
--deploy-mode cluster \
./examples/jars/spark-examples_2.12-3.2.0.jar 1000
```
This command example assumes that you have downloaded a Spark binary distribution, which can be found at [Download Apache Spark](https://spark.apache.org/downloads.html).
For a complete walkthrough of the process using a custom application, refer to Spark's guide to [Running Spark on Kubernetes](https://spark.apache.org/docs/latest/running-on-kubernetes.html).
> Be aware that it is currently not possible to submit an application to a standalone cluster if RPC authentication is configured. [Learn more about the issue](https://issues.apache.org/jira/browse/SPARK-25078).
### Configuring Spark Master as reverse proxy
Spark offers configuration to enable running Spark Master as reverse proxy for worker and application UIs. This can be useful as the Spark Master UI may otherwise use private IPv4 addresses for links to Spark workers and Spark apps.
Coupled with `ingress` configuration, you can set `master.configOptions` and `worker.configOptions` to tell Spark to reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts:
```yaml
master:
configOptions:
-Dspark.ui.reverseProxy=true
-Dspark.ui.reverseProxyUrl=https://spark.your-domain.com
worker:
configOptions:
-Dspark.ui.reverseProxy=true
-Dspark.ui.reverseProxyUrl=https://spark.your-domain.com
ingress:
enabled: true
hostname: spark.your-domain.com
```
See the [Spark Configuration](https://spark.apache.org/docs/latest/configuration.html) docs for detail on the parameters.
### Configure security for Apache Spark
### Configure SSL communication
In order to enable secure transport between workers and master, deploy the Helm chart with the `ssl.enabled=true` chart parameter.
### Create certificate and password secrets
It is necessary to create two secrets for the passwords and certificates. The names of the two secrets should be configured using the `security.passwordsSecretName` and `security.ssl.existingSecret` chart parameters.
#### Create certificates and the certificate secret
To generate the certificates secret, first generate the two certificates and rename them to `spark-keystore.jks` and `spark-truststore.jks`. Use [this script to generate certificates](https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh) for test purposes if required.
Once the certificates are created, create a secret for them with the file names as keys. The keys must be named `spark-keystore.jks` and `spark-truststore.jks`, and the content must be text in JKS format.
#### Create the password secret
The secret for passwords should have three keys: `rpc-authentication-secret`, `ssl-keystore-password` and `ssl-truststore-password`.
#### Configure the chart
Once the secrets are created, configure the chart and set the various security-related parameters, including the `security.certificatesSecretName` and `security.passwordsSecretName` parameters referencing the secrets created previously. Here is an example configuration for chart deployment:
```text
security.certificatesSecretName=my-secret
security.passwordsSecretName=my-passwords-secret
security.rpc.authenticationEnabled=true
security.rpc.encryptionEnabled=true
security.storageEncrytionEnabled=true
security.ssl.enabled=true
security.ssl.needClientAuth=true
```
> NOTE: It is currently not possible to submit an application to a standalone cluster if RPC authentication is configured. [Learn more about this issue](https://issues.apache.org/jira/browse/SPARK-25078).
### Set Pod affinity
This chart allows you to set your custom affinity using the `XXX.affinity` parameter(s). Find more information about Pod affinity in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity).
As an alternative, you can use the preset configurations for pod affinity, pod anti-affinity, and node affinity available at the [bitnami/common](https://github.com/bitnami/charts/tree/main/bitnami/common#affinities) chart. To do so, set the `XXX.podAffinityPreset`, `XXX.podAntiAffinityPreset`, or `XXX.nodeAffinityPreset` parameters.
## Parameters
### Global parameters
| Name | Description | Value |
| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ |
| `global.imageRegistry` | Global Docker image registry | `""` |
| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` |
| `global.defaultStorageClass` | Global default StorageClass for Persistent Volume(s) | `""` |
| `global.storageClass` | DEPRECATED: use global.defaultStorageClass instead | `""` |
| `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `auto` |
### Common parameters
| Name | Description | Value |
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- |
| `kubeVersion` | Force target Kubernetes version (using Helm capabilities if not set) | `""` |
| `nameOverride` | String to partially override common.names.fullname template (will maintain the release name) | `""` |
| `fullnameOverride` | String to fully override common.names.fullname template | `""` |
| `namespaceOverride` | String to fully override common.names.namespace | `""` |
| `commonLabels` | Labels to add to all deployed objects | `{}` |
| `commonAnnotations` | Annotations to add to all deployed objects | `{}` |
| `clusterDomain` | Kubernetes cluster domain name | `cluster.local` |
| `extraDeploy` | Array of extra objects to deploy with the release | `[]` |
| `initScripts` | Dictionary of init scripts. Evaluated as a template. | `{}` |
| `initScriptsCM` | ConfigMap with the init scripts. Evaluated as a template. | `""` |
| `initScriptsSecret` | Secret containing `/docker-entrypoint-initdb.d` scripts to be executed at initialization time that contain sensitive data. Evaluated as a template. | `""` |
| `diagnosticMode.enabled` | Enable diagnostic mode (all probes will be disabled and the command will be overridden) | `false` |
| `diagnosticMode.command` | Command to override all containers in the deployment | `["sleep"]` |
| `diagnosticMode.args` | Args to override all containers in the deployment | `["infinity"]` |
### Spark parameters
| Name | Description | Value |
| ------------------- | ----------------------------------------------------------------------------------------------------- | ----------------------- |
| `image.registry` | Spark image registry | `REGISTRY_NAME` |
| `image.repository` | Spark image repository | `REPOSITORY_NAME/spark` |
| `image.digest` | Spark image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` |
| `image.pullPolicy` | Spark image pull policy | `IfNotPresent` |
| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` |
| `image.debug` | Enable image debug mode | `false` |
| `hostNetwork` | Enable HOST Network | `false` |
### Spark master parameters
| Name | Description | Value |
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
| `master.existingConfigmap` | The name of an existing ConfigMap with your custom configuration for master | `""` |
| `master.containerPorts.http` | Specify the port where the web interface will listen on the master over HTTP | `8080` |
| `master.containerPorts.https` | Specify the port where the web interface will listen on the master over HTTPS | `8480` |
| `master.containerPorts.cluster` | Specify the port where the master listens to communicate with workers | `7077` |
| `master.automountServiceAccountToken` | Mount Service Account token in pod | `false` |
| `master.hostAliases` | Deployment pod host aliases | `[]` |
| `master.extraContainerPorts` | Specify the port where the running jobs inside the masters listens | `[]` |
| `master.daemonMemoryLimit` | Set the memory limit for the master daemon | `""` |
| `master.configOptions` | Use a string to set the config options for in the form "-Dx=y" | `""` |
| `master.extraEnvVars` | Extra environment variables to pass to the master container | `[]` |
| `master.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for master nodes | `""` |
| `master.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for master nodes | `""` |
| `master.podSecurityContext.enabled` | Enable security context | `true` |
| `master.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy | `Always` |
| `master.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface | `[]` |
| `master.podSecurityContext.supplementalGroups` | Set filesystem extra groups | `[]` |
| `master.podSecurityContext.fsGroup` | Set master pod's Security Context Group ID | `1001` |
| `master.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` |
| `master.containerSecurityContext.seLinuxOptions` | Set SELinux options in container | `nil` |
| `master.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` |
| `master.containerSecurityContext.runAsGroup` | Set containers' Security Context runAsGroup | `1001` |
| `master.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` |
| `master.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` |
| `master.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `true` |
| `master.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` |
| `master.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` |
| `master.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` |
| `master.command` | Override default container command (useful when using custom images) | `[]` |
| `master.args` | Override default container args (useful when using custom images) | `[]` |
| `master.podAnnotations` | Annotations for pods in StatefulSet | `{}` |
| `master.podLabels` | Extra labels for pods in StatefulSet | `{}` |
| `master.podAffinityPreset` | Spark master pod affinity preset. Ignored if `master.affinity` is set. Allowed values: `soft` or `hard` | `""` |
| `master.podAntiAffinityPreset` | Spark master pod anti-affinity preset. Ignored if `master.affinity` is set. Allowed values: `soft` or `hard` | `soft` |
| `master.nodeAffinityPreset.type` | Spark master node affinity preset type. Ignored if `master.affinity` is set. Allowed values: `soft` or `hard` | `""` |
| `master.nodeAffinityPreset.key` | Spark master node label key to match Ignored if `master.affinity` is set. | `""` |
| `master.nodeAffinityPreset.values` | Spark master node label values to match. Ignored if `master.affinity` is set. | `[]` |
| `master.affinity` | Spark master affinity for pod assignment | `{}` |
| `master.nodeSelector` | Spark master node labels for pod assignment | `{}` |
| `master.tolerations` | Spark master tolerations for pod assignment | `[]` |
| `master.updateStrategy.type` | Master statefulset strategy type. | `RollingUpdate` |
| `master.priorityClassName` | master pods' priorityClassName | `""` |
| `master.topologySpreadConstraints` | Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template | `[]` |
| `master.schedulerName` | Name of the k8s scheduler (other than default) for master pods | `""` |
| `master.terminationGracePeriodSeconds` | Seconds Redmine pod needs to terminate gracefully | `""` |
| `master.lifecycleHooks` | for the master container(s) to automate configuration before or after startup | `{}` |
| `master.extraVolumes` | Optionally specify extra list of additional volumes for the master pod(s) | `[]` |
| `master.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the master container(s) | `[]` |
| `master.extraVolumeClaimTemplates` | Optionally specify extra list of volumesClaimTemplates for the master statefulset | `[]` |
| `master.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if master.resources is set (master.resources is recommended for production). | `small` |
| `master.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` |
| `master.livenessProbe.enabled` | Enable livenessProbe | `true` |
| `master.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `180` |
| `master.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `20` |
| `master.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` |
| `master.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` |
| `master.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` |
| `master.readinessProbe.enabled` | Enable readinessProbe | `true` |
| `master.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `30` |
| `master.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` |
| `master.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` |
| `master.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` |
| `master.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` |
| `master.startupProbe.enabled` | Enable startupProbe | `false` |
| `master.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `30` |
| `master.startupProbe.periodSeconds` | Period seconds for startupProbe | `10` |
| `master.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `5` |
| `master.startupProbe.failureThreshold` | Failure threshold for startupProbe | `6` |
| `master.startupProbe.successThreshold` | Success threshold for startupProbe | `1` |
| `master.customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` |
| `master.customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` |
| `master.customStartupProbe` | Custom startupProbe that overrides the default one | `{}` |
| `master.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` |
| `master.networkPolicy.allowExternal` | Don't require client label for connections | `true` |
| `master.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` |
| `master.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` |
| `master.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy | `[]` |
| `master.networkPolicy.ingressNSMatchLabels` | Labels to match to allow traffic from other namespaces | `{}` |
| `master.networkPolicy.ingressNSPodMatchLabels` | Pod labels to match to allow traffic from other namespaces | `{}` |
| `master.sidecars` | Add additional sidecar containers to the master pod(s) | `[]` |
| `master.initContainers` | Add initContainers to the master pods. | `[]` |
| `master.pdb.create` | Enable/disable a Pod Disruption Budget creation | `true` |
| `master.pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `""` |
| `master.pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `master.pdb.minAvailable` and `master.pdb.maxUnavailable` are empty. | `""` |
### Spark worker parameters
| Name | Description | Value |
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
| `worker.existingConfigmap` | The name of an existing ConfigMap with your custom configuration for workers | `""` |
| `worker.containerPorts.http` | Specify the port where the web interface will listen on the worker over HTTP | `8080` |
| `worker.containerPorts.https` | Specify the port where the web interface will listen on the worker over HTTPS | `8480` |
| `worker.containerPorts.cluster` | Specify the port where the worker listens to communicate with workers | `""` |
| `worker.automountServiceAccountToken` | Mount Service Account token in pod | `false` |
| `worker.hostAliases` | Add deployment host aliases | `[]` |
| `worker.extraContainerPorts` | Specify the port where the running jobs inside the workers listens | `[]` |
| `worker.daemonMemoryLimit` | Set the memory limit for the worker daemon | `""` |
| `worker.memoryLimit` | Set the maximum memory the worker is allowed to use | `""` |
| `worker.coreLimit` | Se the maximum number of cores that the worker can use | `""` |
| `worker.dir` | Set a custom working directory for the application | `""` |
| `worker.javaOptions` | Set options for the JVM in the form `-Dx=y` | `""` |
| `worker.configOptions` | Set extra options to configure the worker in the form `-Dx=y` | `""` |
| `worker.extraEnvVars` | An array to add extra env vars | `[]` |
| `worker.extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for worker nodes | `""` |
| `worker.extraEnvVarsSecret` | Name of existing Secret containing extra env vars for worker nodes | `""` |
| `worker.replicaCount` | Number of spark workers (will be the minimum number when autoscaling is enabled) | `2` |
| `worker.podSecurityContext.enabled` | Enable security context | `true` |
| `worker.podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy | `Always` |
| `worker.podSecurityContext.sysctls` | Set kernel settings using the sysctl interface | `[]` |
| `worker.podSecurityContext.supplementalGroups` | Set filesystem extra groups | `[]` |
| `worker.podSecurityContext.fsGroup` | Group ID for the container | `1001` |
| `worker.podSecurityContext.seLinuxOptions` | SELinux options for the container | `nil` |
| `worker.containerSecurityContext.enabled` | Enabled containers' Security Context | `true` |
| `worker.containerSecurityContext.seLinuxOptions` | Set SELinux options in container | `nil` |
| `worker.containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` |
| `worker.containerSecurityContext.runAsGroup` | Set containers' Security Context runAsGroup | `1001` |
| `worker.containerSecurityContext.runAsNonRoot` | Set container's Security Context runAsNonRoot | `true` |
| `worker.containerSecurityContext.privileged` | Set container's Security Context privileged | `false` |
| `worker.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `true` |
| `worker.containerSecurityContext.allowPrivilegeEscalation` | Set container's Security Context allowPrivilegeEscalation | `false` |
| `worker.containerSecurityContext.capabilities.drop` | List of capabilities to be dropped | `["ALL"]` |
| `worker.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` |
| `worker.command` | Override default container command (useful when using custom images) | `[]` |
| `worker.args` | Override default container args (useful when using custom images) | `[]` |
| `worker.podAnnotations` | Annotations for pods in StatefulSet | `{}` |
| `worker.podLabels` | Extra labels for pods in StatefulSet | `{}` |
| `worker.podAffinityPreset` | Spark worker pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` | `""` |
| `worker.podAntiAffinityPreset` | Spark worker pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` | `soft` |
| `worker.nodeAffinityPreset.type` | Spark worker node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard` | `""` |
| `worker.nodeAffinityPreset.key` | Spark worker node label key to match Ignored if `worker.affinity` is set. | `""` |
| `worker.nodeAffinityPreset.values` | Spark worker node label values to match. Ignored if `worker.affinity` is set. | `[]` |
| `worker.affinity` | Spark worker affinity for pod assignment | `{}` |
| `worker.nodeSelector` | Spark worker node labels for pod assignment | `{}` |
| `worker.tolerations` | Spark worker tolerations for pod assignment | `[]` |
| `worker.updateStrategy.type` | Worker statefulset strategy type. | `RollingUpdate` |
| `worker.podManagementPolicy` | Statefulset Pod Management Policy Type | `OrderedReady` |
| `worker.priorityClassName` | worker pods' priorityClassName | `""` |
| `worker.topologySpreadConstraints` | Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template | `[]` |
| `worker.schedulerName` | Name of the k8s scheduler (other than default) for worker pods | `""` |
| `worker.terminationGracePeriodSeconds` | Seconds Redmine pod needs to terminate gracefully | `""` |
| `worker.lifecycleHooks` | for the worker container(s) to automate configuration before or after startup | `{}` |
| `worker.extraVolumes` | Optionally specify extra list of additional volumes for the worker pod(s) | `[]` |
| `worker.extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the master container(s) | `[]` |
| `worker.extraVolumeClaimTemplates` | Optionally specify extra list of volumesClaimTemplates for the worker statefulset | `[]` |
| `worker.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if worker.resources is set (worker.resources is recommended for production). | `small` |
| `worker.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` |
| `worker.livenessProbe.enabled` | Enable livenessProbe | `true` |
| `worker.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `180` |
| `worker.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `20` |
| `worker.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` |
| `worker.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` |
| `worker.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` |
| `worker.readinessProbe.enabled` | Enable readinessProbe | `true` |
| `worker.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `30` |
| `worker.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` |
| `worker.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` |
| `worker.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` |
| `worker.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` |
| `worker.startupProbe.enabled` | Enable startupProbe | `true` |
| `worker.startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `30` |
| `worker.startupProbe.periodSeconds` | Period seconds for startupProbe | `10` |
| `worker.startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `5` |
| `worker.startupProbe.failureThreshold` | Failure threshold for startupProbe | `6` |
| `worker.startupProbe.successThreshold` | Success threshold for startupProbe | `1` |
| `worker.customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` |
| `worker.customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` |
| `worker.customStartupProbe` | Custom startupProbe that overrides the default one | `{}` |
| `worker.networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` |
| `worker.networkPolicy.allowExternal` | Don't require client label for connections | `true` |
| `worker.networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` |
| `worker.networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolicy | `[]` |
| `worker.networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy | `[]` |
| `worker.networkPolicy.ingressNSMatchLabels` | Labels to match to allow traffic from other namespaces | `{}` |
| `worker.networkPolicy.ingressNSPodMatchLabels` | Pod labels to match to allow traffic from other namespaces | `{}` |
| `worker.sidecars` | Add additional sidecar containers to the worker pod(s) | `[]` |
| `worker.initContainers` | Add initContainers to the worker pods. | `[]` |
| `worker.pdb.create` | Enable/disable a Pod Disruption Budget creation | `true` |
| `worker.pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `""` |
| `worker.pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable. Defaults to `1` if both `worker.pdb.minAvailable` and `worker.pdb.maxUnavailable` are empty. | `""` |
| `worker.autoscaling.enabled` | Enable replica autoscaling depending on CPU | `false` |
| `worker.autoscaling.minReplicas` | Minimum number of worker replicas | `""` |
| `worker.autoscaling.maxReplicas` | Maximum number of worker replicas | `5` |
| `worker.autoscaling.targetCPU` | Target CPU utilization percentage | `50` |
| `worker.autoscaling.targetMemory` | Target Memory utilization percentage | `""` |
### Security parameters
| Name | Description | Value |
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
| `security.passwordsSecretName` | Name of the secret that contains all the passwords | `""` |
| `security.rpc.authenticationEnabled` | Enable the RPC authentication | `false` |
| `security.rpc.encryptionEnabled` | Enable the encryption for RPC | `false` |
| `security.storageEncryptionEnabled` | Enables local storage encryption | `false` |
| `security.certificatesSecretName` | Name of the secret that contains the certificates. | `""` |
| `security.ssl.enabled` | Enable the SSL configuration | `false` |
| `security.ssl.needClientAuth` | Enable the client authentication | `false` |
| `security.ssl.protocol` | Set the SSL protocol | `TLSv1.2` |
| `security.ssl.existingSecret` | Name of the existing secret containing the TLS certificates | `""` |
| `security.ssl.autoGenerated` | Create self-signed TLS certificates. Currently only supports PEM certificates | `false` |
| `security.ssl.keystorePassword` | Set the password of the JKS Keystore | `""` |
| `security.ssl.truststorePassword` | Truststore password. | `""` |
| `security.ssl.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if security.ssl.resources is set (security.ssl.resources is recommended for production). | `small` |
| `security.ssl.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` |
### Traffic Exposure parameters
| Name | Description | Value |
| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------ |
| `service.type` | Kubernetes Service type | `ClusterIP` |
| `service.ports.http` | Spark client port for HTTP | `80` |
| `service.ports.https` | Spark client port for HTTPS | `443` |
| `service.ports.cluster` | Spark cluster port | `7077` |
| `service.nodePorts.http` | Kubernetes web node port for HTTP | `""` |
| `service.nodePorts.https` | Kubernetes web node port for HTTPS | `""` |
| `service.nodePorts.cluster` | Kubernetes cluster node port | `""` |
| `service.clusterIP` | Spark service Cluster IP | `""` |
| `service.loadBalancerIP` | Load balancer IP if spark service type is `LoadBalancer` | `""` |
| `service.loadBalancerSourceRanges` | Spark service Load Balancer sources | `[]` |
| `service.externalTrafficPolicy` | Spark service external traffic policy | `Cluster` |
| `service.annotations` | Additional custom annotations for Spark service | `{}` |
| `service.extraPorts` | Extra ports to expose in Spark service (normally used with the `sidecars` value) | `[]` |
| `service.sessionAffinity` | Control where client requests go, to the same pod or round-robin | `None` |
| `service.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` |
| `service.headless.annotations` | Annotations for the headless service. | `{}` |
| `ingress.enabled` | Enable ingress controller resource | `false` |
| `ingress.pathType` | Ingress path type | `ImplementationSpecific` |
| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` |
| `ingress.hostname` | Default host for the ingress resource | `spark.local` |
| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `""` |
| `ingress.path` | The Path to Spark. You may need to set this to '/*' in order to use this with ALB ingress controllers. | `/` |
| `ingress.annotations` | Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations. | `{}` |
| `ingress.tls` | Enable TLS configuration for the hostname defined at ingress.hostname parameter | `false` |
| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` |
| `ingress.extraHosts` | The list of additional hostnames to be covered with this ingress record. | `[]` |
| `ingress.extraPaths` | Any additional arbitrary paths that may need to be added to the ingress under the main host. | `[]` |
| `ingress.extraTls` | The tls configuration for additional hostnames to be covered with this ingress record. | `[]` |
| `ingress.secrets` | If you're providing your own certificates, please use this to add the certificates as secrets | `[]` |
| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` |
### Other parameters
| Name | Description | Value |
| --------------------------------------------- | ------------------------------------------------------ | ------- |
| `serviceAccount.create` | Enable the creation of a ServiceAccount for Spark pods | `true` |
| `serviceAccount.name` | The name of the ServiceAccount to use. | `""` |
| `serviceAccount.annotations` | Annotations for Spark Service Account | `{}` |
| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a service account. | `false` |
### Metrics parameters
| Name | Description | Value |
| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | ------- |
| `metrics.enabled` | Start a side-car prometheus exporter | `false` |
| `metrics.masterAnnotations` | Annotations for the Prometheus metrics on master nodes | `{}` |
| `metrics.workerAnnotations` | Annotations for the Prometheus metrics on worker nodes | `{}` |
| `metrics.podMonitor.enabled` | If the operator is installed in your cluster, set to true to create a PodMonitor Resource for scraping metrics using PrometheusOperator | `false` |
| `metrics.podMonitor.extraMetricsEndpoints` | Add metrics endpoints for monitoring the jobs running in the worker nodes | `[]` |
| `metrics.podMonitor.namespace` | Specify the namespace in which the podMonitor resource will be created | `""` |
| `metrics.podMonitor.interval` | Specify the interval at which metrics should be scraped | `30s` |
| `metrics.podMonitor.scrapeTimeout` | Specify the timeout after which the scrape is ended | `""` |
| `metrics.podMonitor.additionalLabels` | Additional labels that can be used so PodMonitors will be discovered by Prometheus | `{}` |
| `metrics.prometheusRule.enabled` | Set this to true to create prometheusRules for Prometheus | `false` |
| `metrics.prometheusRule.namespace` | Namespace where the prometheusRules resource should be created | `""` |
| `metrics.prometheusRule.additionalLabels` | Additional labels that can be used so prometheusRules will be discovered by Prometheus | `{}` |
| `metrics.prometheusRule.rules` | Custom Prometheus [rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) | `[]` |
Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
```console
helm install my-release \
--set master.webPort=8081 oci://REGISTRY_NAME/REPOSITORY_NAME/spark
```
> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`.
The above command sets the spark master web port to `8081`.
Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example,
```console
helm install my-release -f values.yaml oci://REGISTRY_NAME/REPOSITORY_NAME/spark
```
> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`.
> **Tip**: You can use the default [values.yaml](https://github.com/bitnami/charts/tree/main/bitnami/spark/values.yaml)
## Troubleshooting
Find more information about how to deal with common errors related to Bitnami's Helm charts in [this troubleshooting guide](https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues).
## Upgrading
### To 9.0.0
This major bump changes the following security defaults:
- `runAsGroup` is changed from `0` to `1001`
- `readOnlyRootFilesystem` is set to `true`
- `resourcesPreset` is changed from `none` to the minimum size working in our test suites (NOTE: `resourcesPreset` is not meant for production usage, but `resources` adapted to your use case).
- `global.compatibility.openshift.adaptSecurityContext` is changed from `disabled` to `auto`.
This could potentially break any customization or init scripts used in your deployment. If this is the case, change the default values to the previous ones.
### To 6.0.0
This chart major version standarizes the chart templates and values, modifying some existing parameters names and adding several more. These parameter modifications can be sumarised in the following:
- `worker.autoscaling.CpuTargetPercentage/.replicasMax` parameters are now found by `worker.autoscaling.targetCPU/.maxReplicas`.
- `webport/webPortHttps/cluster` parameters are now found by `containerPorts.http/.https/.cluster`.
- `service.webport/webPortHttps/cluster` parameters are now found by `service.ports.http/.https/.cluster`.
- `service.nodePorts.web/webHttps/cluster` parameters are now found by `service.nodePorts.http/.https/.cluster`.
- `xxxxx.securityContext` parameters are now under `xxxxx.podSecurityContext`
- `xxxxx.configurationConfigMap` parameter has been renamed to `xxxxx.existingConfigmap`
- `extraPodLabels` parameter has been renamed to `podLabels`
Besides the changes detailed above, no issues are expected to appear when upgrading.
### To 5.0.0
This version standardizes the way of defining Ingress rules. When configuring a single hostname for the Ingress rule, set the `ingress.hostname` value. When defining more than one, set the `ingress.extraHosts` array. Apart from this case, no issues are expected to appear when upgrading.
### To 4.0.0
[On November 13, 2020, Helm v2 support formally ended](https://github.com/helm/charts#status-of-the-project). This major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL.
### To 3.0.0
- This version introduces `bitnami/common`, a [library chart](https://helm.sh/docs/topics/library_charts/#helm) as a dependency. More documentation about this new utility could be found [here](https://github.com/bitnami/charts/tree/main/bitnami/common#bitnami-common-library-chart). Please, make sure that you have updated the chart dependencies before executing any upgrade.
- Spark container images are updated to use Hadoop `3.2.x`: [Notable Changes: 3.0.0-debian-10-r44](https://github.com/bitnami/containers/tree/main/bitnami/spark#300-debian-10-r44)
> Note: Backwards compatibility is not guaranteed due to the above mentioned changes. Please make sure your workloads are compatible with the new version of Hadoop before upgrading. Backups are always recommended before any upgrade operation.
## License
Copyright &copy; 2024 Broadcom. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
<http://www.apache.org/licenses/LICENSE-2.0>
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/.helmignore
Executable
+26
View File
@@ -0,0 +1,26 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# img folder
img/
# Changelog
CHANGELOG.md
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/Chart.yaml
Executable
+23
View File
@@ -0,0 +1,23 @@
annotations:
category: Infrastructure
licenses: Apache-2.0
apiVersion: v2
appVersion: 2.22.0
description: A Library Helm Chart for grouping common logic between bitnami charts.
This chart is not deployable by itself.
home: https://bitnami.com
icon: https://bitnami.com/downloads/logos/bitnami-mark.png
keywords:
- common
- helper
- template
- function
- bitnami
maintainers:
- name: Broadcom, Inc. All Rights Reserved.
url: https://github.com/bitnami/charts
name: common
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/common
type: library
version: 2.22.0
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/README.md
Executable
+235
View File
@@ -0,0 +1,235 @@
# Bitnami Common Library Chart
A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between Bitnami charts.
## TL;DR
```yaml
dependencies:
- name: common
version: 2.x.x
repository: oci://registry-1.docker.io/bitnamicharts
```
```console
helm dependency update
```
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "common.names.fullname" . }}
data:
myvalue: "Hello World"
```
Looking to use our applications in production? Try [VMware Tanzu Application Catalog](https://bitnami.com/enterprise), the commercial edition of the Bitnami catalog.
## Introduction
This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager.
Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters.
## Prerequisites
- Kubernetes 1.23+
- Helm 3.8.0+
## Parameters
## Special input schemas
### ImageRoot
```yaml
registry:
type: string
description: Docker registry where the image is located
example: docker.io
repository:
type: string
description: Repository and image name
example: bitnami/nginx
tag:
type: string
description: image tag
example: 1.16.1-debian-10-r63
pullPolicy:
type: string
description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
pullSecrets:
type: array
items:
type: string
description: Optionally specify an array of imagePullSecrets (evaluated as templates).
debug:
type: boolean
description: Set to true if you would like to see extra information on logs
example: false
## An instance would be:
# registry: docker.io
# repository: bitnami/nginx
# tag: 1.16.1-debian-10-r63
# pullPolicy: IfNotPresent
# debug: false
```
### Persistence
```yaml
enabled:
type: boolean
description: Whether enable persistence.
example: true
storageClass:
type: string
description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning.
example: "-"
accessMode:
type: string
description: Access mode for the Persistent Volume Storage.
example: ReadWriteOnce
size:
type: string
description: Size the Persistent Volume Storage.
example: 8Gi
path:
type: string
description: Path to be persisted.
example: /bitnami
## An instance would be:
# enabled: true
# storageClass: "-"
# accessMode: ReadWriteOnce
# size: 8Gi
# path: /bitnami
```
### ExistingSecret
```yaml
name:
type: string
description: Name of the existing secret.
example: mySecret
keyMapping:
description: Mapping between the expected key name and the name of the key in the existing secret.
type: object
## An instance would be:
# name: mySecret
# keyMapping:
# password: myPasswordKey
```
#### Example of use
When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets.
```yaml
# templates/secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
name: {{ include "common.names.fullname" . }}
labels:
app: {{ include "common.names.fullname" . }}
type: Opaque
data:
password: {{ .Values.password | b64enc | quote }}
# templates/dpl.yaml
---
...
env:
- name: PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }}
key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }}
...
# values.yaml
---
name: mySecret
keyMapping:
password: myPasswordKey
```
### ValidateValue
#### NOTES.txt
```console
{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}}
{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}}
{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }}
```
If we force those values to be empty we will see some alerts
```console
helm install test mychart --set path.to.value00="",path.to.value01=""
'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value:
export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d)
'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value:
export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d)
```
## Upgrading
### To 1.0.0
[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL.
#### What changes were introduced in this major version?
- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field.
- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information.
- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts
#### Considerations when upgrading to this version
- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues
- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore
- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3
#### Useful links
- <https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-resolve-helm2-helm3-post-migration-issues-index.html>
- <https://helm.sh/docs/topics/v2_v3_migration/>
- <https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/>
## License
Copyright &copy; 2024 Broadcom. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
<http://www.apache.org/licenses/LICENSE-2.0>
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_affinities.tpl
Executable
+139
View File
@@ -0,0 +1,139 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return a soft nodeAffinity definition
{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}}
*/}}
{{- define "common.affinities.nodes.soft" -}}
preferredDuringSchedulingIgnoredDuringExecution:
- preference:
matchExpressions:
- key: {{ .key }}
operator: In
values:
{{- range .values }}
- {{ . | quote }}
{{- end }}
weight: 1
{{- end -}}
{{/*
Return a hard nodeAffinity definition
{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}}
*/}}
{{- define "common.affinities.nodes.hard" -}}
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: {{ .key }}
operator: In
values:
{{- range .values }}
- {{ . | quote }}
{{- end }}
{{- end -}}
{{/*
Return a nodeAffinity definition
{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}}
*/}}
{{- define "common.affinities.nodes" -}}
{{- if eq .type "soft" }}
{{- include "common.affinities.nodes.soft" . -}}
{{- else if eq .type "hard" }}
{{- include "common.affinities.nodes.hard" . -}}
{{- end -}}
{{- end -}}
{{/*
Return a topologyKey definition
{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}}
*/}}
{{- define "common.affinities.topologyKey" -}}
{{ .topologyKey | default "kubernetes.io/hostname" -}}
{{- end -}}
{{/*
Return a soft podAffinity/podAntiAffinity definition
{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}}
*/}}
{{- define "common.affinities.pods.soft" -}}
{{- $component := default "" .component -}}
{{- $customLabels := default (dict) .customLabels -}}
{{- $extraMatchLabels := default (dict) .extraMatchLabels -}}
{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}}
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }}
{{- if not (empty $component) }}
{{ printf "app.kubernetes.io/component: %s" $component }}
{{- end }}
{{- range $key, $value := $extraMatchLabels }}
{{ $key }}: {{ $value | quote }}
{{- end }}
topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }}
weight: 1
{{- range $extraPodAffinityTerms }}
- podAffinityTerm:
labelSelector:
matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }}
{{- if not (empty $component) }}
{{ printf "app.kubernetes.io/component: %s" $component }}
{{- end }}
{{- range $key, $value := .extraMatchLabels }}
{{ $key }}: {{ $value | quote }}
{{- end }}
topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }}
weight: {{ .weight | default 1 -}}
{{- end -}}
{{- end -}}
{{/*
Return a hard podAffinity/podAntiAffinity definition
{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}}
*/}}
{{- define "common.affinities.pods.hard" -}}
{{- $component := default "" .component -}}
{{- $customLabels := default (dict) .customLabels -}}
{{- $extraMatchLabels := default (dict) .extraMatchLabels -}}
{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}}
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }}
{{- if not (empty $component) }}
{{ printf "app.kubernetes.io/component: %s" $component }}
{{- end }}
{{- range $key, $value := $extraMatchLabels }}
{{ $key }}: {{ $value | quote }}
{{- end }}
topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }}
{{- range $extraPodAffinityTerms }}
- labelSelector:
matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }}
{{- if not (empty $component) }}
{{ printf "app.kubernetes.io/component: %s" $component }}
{{- end }}
{{- range $key, $value := .extraMatchLabels }}
{{ $key }}: {{ $value | quote }}
{{- end }}
topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }}
{{- end -}}
{{- end -}}
{{/*
Return a podAffinity/podAntiAffinity definition
{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}}
*/}}
{{- define "common.affinities.pods" -}}
{{- if eq .type "soft" }}
{{- include "common.affinities.pods.soft" . -}}
{{- else if eq .type "hard" }}
{{- include "common.affinities.pods.hard" . -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_capabilities.tpl
Executable
+229
View File
@@ -0,0 +1,229 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return the target Kubernetes version
*/}}
{{- define "common.capabilities.kubeVersion" -}}
{{- default (default .Capabilities.KubeVersion.Version .Values.kubeVersion) ((.Values.global).kubeVersion) -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for poddisruptionbudget.
*/}}
{{- define "common.capabilities.policy.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.21-0" $kubeVersion) -}}
{{- print "policy/v1beta1" -}}
{{- else -}}
{{- print "policy/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for networkpolicy.
*/}}
{{- define "common.capabilities.networkPolicy.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.7-0" $kubeVersion) -}}
{{- print "extensions/v1beta1" -}}
{{- else -}}
{{- print "networking.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for cronjob.
*/}}
{{- define "common.capabilities.cronjob.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.21-0" $kubeVersion) -}}
{{- print "batch/v1beta1" -}}
{{- else -}}
{{- print "batch/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for daemonset.
*/}}
{{- define "common.capabilities.daemonset.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.14-0" $kubeVersion) -}}
{{- print "extensions/v1beta1" -}}
{{- else -}}
{{- print "apps/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for deployment.
*/}}
{{- define "common.capabilities.deployment.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.14-0" $kubeVersion) -}}
{{- print "extensions/v1beta1" -}}
{{- else -}}
{{- print "apps/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for statefulset.
*/}}
{{- define "common.capabilities.statefulset.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.14-0" $kubeVersion) -}}
{{- print "apps/v1beta1" -}}
{{- else -}}
{{- print "apps/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for ingress.
*/}}
{{- define "common.capabilities.ingress.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if (.Values.ingress).apiVersion -}}
{{- .Values.ingress.apiVersion -}}
{{- else if and (not (empty $kubeVersion)) (semverCompare "<1.14-0" $kubeVersion) -}}
{{- print "extensions/v1beta1" -}}
{{- else if and (not (empty $kubeVersion)) (semverCompare "<1.19-0" $kubeVersion) -}}
{{- print "networking.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "networking.k8s.io/v1" -}}
{{- end }}
{{- end -}}
{{/*
Return the appropriate apiVersion for RBAC resources.
*/}}
{{- define "common.capabilities.rbac.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.17-0" $kubeVersion) -}}
{{- print "rbac.authorization.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "rbac.authorization.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for CRDs.
*/}}
{{- define "common.capabilities.crd.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.19-0" $kubeVersion) -}}
{{- print "apiextensions.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "apiextensions.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for APIService.
*/}}
{{- define "common.capabilities.apiService.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.10-0" $kubeVersion) -}}
{{- print "apiregistration.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "apiregistration.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for Horizontal Pod Autoscaler.
*/}}
{{- define "common.capabilities.hpa.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" .context -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.23-0" $kubeVersion) -}}
{{- if .beta2 -}}
{{- print "autoscaling/v2beta2" -}}
{{- else -}}
{{- print "autoscaling/v2beta1" -}}
{{- end -}}
{{- else -}}
{{- print "autoscaling/v2" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for Vertical Pod Autoscaler.
*/}}
{{- define "common.capabilities.vpa.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" .context -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.23-0" $kubeVersion) -}}
{{- if .beta2 -}}
{{- print "autoscaling/v2beta2" -}}
{{- else -}}
{{- print "autoscaling/v2beta1" -}}
{{- end -}}
{{- else -}}
{{- print "autoscaling/v2" -}}
{{- end -}}
{{- end -}}
{{/*
Returns true if PodSecurityPolicy is supported
*/}}
{{- define "common.capabilities.psp.supported" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if or (empty $kubeVersion) (semverCompare "<1.25-0" $kubeVersion) -}}
{{- true -}}
{{- end -}}
{{- end -}}
{{/*
Returns true if AdmissionConfiguration is supported
*/}}
{{- define "common.capabilities.admissionConfiguration.supported" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if or (empty $kubeVersion) (not (semverCompare "<1.23-0" $kubeVersion)) -}}
{{- true -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for AdmissionConfiguration.
*/}}
{{- define "common.capabilities.admissionConfiguration.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.23-0" $kubeVersion) -}}
{{- print "apiserver.config.k8s.io/v1alpha1" -}}
{{- else if and (not (empty $kubeVersion)) (semverCompare "<1.25-0" $kubeVersion) -}}
{{- print "apiserver.config.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "apiserver.config.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Return the appropriate apiVersion for PodSecurityConfiguration.
*/}}
{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}}
{{- $kubeVersion := include "common.capabilities.kubeVersion" . -}}
{{- if and (not (empty $kubeVersion)) (semverCompare "<1.23-0" $kubeVersion) -}}
{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}}
{{- else if and (not (empty $kubeVersion)) (semverCompare "<1.25-0" $kubeVersion) -}}
{{- print "pod-security.admission.config.k8s.io/v1beta1" -}}
{{- else -}}
{{- print "pod-security.admission.config.k8s.io/v1" -}}
{{- end -}}
{{- end -}}
{{/*
Returns true if the used Helm version is 3.3+.
A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure.
This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error.
**To be removed when the catalog's minimun Helm version is 3.3**
*/}}
{{- define "common.capabilities.supportsHelmVersion" -}}
{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }}
{{- true -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_compatibility.tpl
Executable
+42
View File
@@ -0,0 +1,42 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return true if the detected platform is Openshift
Usage:
{{- include "common.compatibility.isOpenshift" . -}}
*/}}
{{- define "common.compatibility.isOpenshift" -}}
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1" -}}
{{- true -}}
{{- end -}}
{{- end -}}
{{/*
Render a compatible securityContext depending on the platform. By default it is maintained as it is. In other platforms like Openshift we remove default user/group values that do not work out of the box with the restricted-v1 SCC
Usage:
{{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.containerSecurityContext "context" $) -}}
*/}}
{{- define "common.compatibility.renderSecurityContext" -}}
{{- $adaptedContext := .secContext -}}
{{- if (((.context.Values.global).compatibility).openshift) -}}
{{- if or (eq .context.Values.global.compatibility.openshift.adaptSecurityContext "force") (and (eq .context.Values.global.compatibility.openshift.adaptSecurityContext "auto") (include "common.compatibility.isOpenshift" .context)) -}}
{{/* Remove incompatible user/group values that do not work in Openshift out of the box */}}
{{- $adaptedContext = omit $adaptedContext "fsGroup" "runAsUser" "runAsGroup" -}}
{{- if not .secContext.seLinuxOptions -}}
{{/* If it is an empty object, we remove it from the resulting context because it causes validation issues */}}
{{- $adaptedContext = omit $adaptedContext "seLinuxOptions" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/* Remove fields that are disregarded when running the container in privileged mode */}}
{{- if $adaptedContext.privileged -}}
{{- $adaptedContext = omit $adaptedContext "capabilities" "seLinuxOptions" -}}
{{- end -}}
{{- omit $adaptedContext "enabled" | toYaml -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_errors.tpl
Executable
+28
View File
@@ -0,0 +1,28 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Through error when upgrading using empty passwords values that must not be empty.
Usage:
{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}}
{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}}
{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }}
Required password params:
- validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error.
- context - Context - Required. Parent context.
*/}}
{{- define "common.errors.upgrade.passwords.empty" -}}
{{- $validationErrors := join "" .validationErrors -}}
{{- if and $validationErrors .context.Release.IsUpgrade -}}
{{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}}
{{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}}
{{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}}
{{- $errorString = print $errorString "\n%s" -}}
{{- printf $errorString $validationErrors | fail -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_images.tpl
Executable
+115
View File
@@ -0,0 +1,115 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return the proper image name.
If image tag and digest are not defined, termination fallbacks to chart appVersion.
{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global "chart" .Chart ) }}
*/}}
{{- define "common.images.image" -}}
{{- $registryName := default .imageRoot.registry ((.global).imageRegistry) -}}
{{- $repositoryName := .imageRoot.repository -}}
{{- $separator := ":" -}}
{{- $termination := .imageRoot.tag | toString -}}
{{- if not .imageRoot.tag }}
{{- if .chart }}
{{- $termination = .chart.AppVersion | toString -}}
{{- end -}}
{{- end -}}
{{- if .imageRoot.digest }}
{{- $separator = "@" -}}
{{- $termination = .imageRoot.digest | toString -}}
{{- end -}}
{{- if $registryName }}
{{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
{{- else -}}
{{- printf "%s%s%s" $repositoryName $separator $termination -}}
{{- end -}}
{{- end -}}
{{/*
Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
*/}}
{{- define "common.images.pullSecrets" -}}
{{- $pullSecrets := list }}
{{- range ((.global).imagePullSecrets) -}}
{{- if kindIs "map" . -}}
{{- $pullSecrets = append $pullSecrets .name -}}
{{- else -}}
{{- $pullSecrets = append $pullSecrets . -}}
{{- end }}
{{- end -}}
{{- range .images -}}
{{- range .pullSecrets -}}
{{- if kindIs "map" . -}}
{{- $pullSecrets = append $pullSecrets .name -}}
{{- else -}}
{{- $pullSecrets = append $pullSecrets . -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- if (not (empty $pullSecrets)) -}}
imagePullSecrets:
{{- range $pullSecrets | uniq }}
- name: {{ . }}
{{- end }}
{{- end }}
{{- end -}}
{{/*
Return the proper Docker Image Registry Secret Names evaluating values as templates
{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }}
*/}}
{{- define "common.images.renderPullSecrets" -}}
{{- $pullSecrets := list }}
{{- $context := .context }}
{{- range (($context.Values.global).imagePullSecrets) -}}
{{- if kindIs "map" . -}}
{{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}}
{{- else -}}
{{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}}
{{- end -}}
{{- end -}}
{{- range .images -}}
{{- range .pullSecrets -}}
{{- if kindIs "map" . -}}
{{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}}
{{- else -}}
{{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- if (not (empty $pullSecrets)) -}}
imagePullSecrets:
{{- range $pullSecrets | uniq }}
- name: {{ . }}
{{- end }}
{{- end }}
{{- end -}}
{{/*
Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion)
{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }}
*/}}
{{- define "common.images.version" -}}
{{- $imageTag := .imageRoot.tag | toString -}}
{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}}
{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}}
{{- $version := semver $imageTag -}}
{{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}}
{{- else -}}
{{- print .chart.AppVersion -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_ingress.tpl
Executable
+73
View File
@@ -0,0 +1,73 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Generate backend entry that is compatible with all Kubernetes API versions.
Usage:
{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }}
Params:
- serviceName - String. Name of an existing service backend
- servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer.
- context - Dict - Required. The context for the template evaluation.
*/}}
{{- define "common.ingress.backend" -}}
{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}}
{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}}
serviceName: {{ .serviceName }}
servicePort: {{ .servicePort }}
{{- else -}}
service:
name: {{ .serviceName }}
port:
{{- if typeIs "string" .servicePort }}
name: {{ .servicePort }}
{{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }}
number: {{ .servicePort | int }}
{{- end }}
{{- end -}}
{{- end -}}
{{/*
Print "true" if the API pathType field is supported
Usage:
{{ include "common.ingress.supportsPathType" . }}
*/}}
{{- define "common.ingress.supportsPathType" -}}
{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}}
{{- print "false" -}}
{{- else -}}
{{- print "true" -}}
{{- end -}}
{{- end -}}
{{/*
Returns true if the ingressClassname field is supported
Usage:
{{ include "common.ingress.supportsIngressClassname" . }}
*/}}
{{- define "common.ingress.supportsIngressClassname" -}}
{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}}
{{- print "false" -}}
{{- else -}}
{{- print "true" -}}
{{- end -}}
{{- end -}}
{{/*
Return true if cert-manager required annotations for TLS signed
certificates are set in the Ingress annotations
Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations
Usage:
{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }}
*/}}
{{- define "common.ingress.certManagerRequest" -}}
{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }}
{{- true -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_labels.tpl
Executable
+46
View File
@@ -0,0 +1,46 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Kubernetes standard labels
{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}}
*/}}
{{- define "common.labels.standard" -}}
{{- if and (hasKey . "customLabels") (hasKey . "context") -}}
{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}}
{{- with .context.Chart.AppVersion -}}
{{- $_ := set $default "app.kubernetes.io/version" . -}}
{{- end -}}
{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }}
{{- else -}}
app.kubernetes.io/name: {{ include "common.names.name" . }}
helm.sh/chart: {{ include "common.names.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Chart.AppVersion }}
app.kubernetes.io/version: {{ . | quote }}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector
{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}}
We don't want to loop over custom labels appending them to the selector
since it's very likely that it will break deployments, services, etc.
However, it's important to overwrite the standard labels if the user
overwrote them on metadata.labels fields.
*/}}
{{- define "common.labels.matchLabels" -}}
{{- if and (hasKey . "customLabels") (hasKey . "context") -}}
{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }}
{{- else -}}
app.kubernetes.io/name: {{ include "common.names.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_names.tpl
Executable
+71
View File
@@ -0,0 +1,71 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "common.names.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "common.names.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "common.names.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create a default fully qualified dependency name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
Usage:
{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }}
*/}}
{{- define "common.names.dependency.fullname" -}}
{{- if .chartValues.fullnameOverride -}}
{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .chartName .chartValues.nameOverride -}}
{{- if contains $name .context.Release.Name -}}
{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Allow the release namespace to be overridden for multi-namespace deployments in combined charts.
*/}}
{{- define "common.names.namespace" -}}
{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a fully qualified app name adding the installation's namespace.
*/}}
{{- define "common.names.fullname.namespace" -}}
{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_resources.tpl
Executable
+50
View File
@@ -0,0 +1,50 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return a resource request/limit object based on a given preset.
These presets are for basic testing and not meant to be used in production
{{ include "common.resources.preset" (dict "type" "nano") -}}
*/}}
{{- define "common.resources.preset" -}}
{{/* The limits are the requests increased by 50% (except ephemeral-storage and xlarge/2xlarge sizes)*/}}
{{- $presets := dict
"nano" (dict
"requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "150m" "memory" "192Mi" "ephemeral-storage" "2Gi")
)
"micro" (dict
"requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "375m" "memory" "384Mi" "ephemeral-storage" "2Gi")
)
"small" (dict
"requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "750m" "memory" "768Mi" "ephemeral-storage" "2Gi")
)
"medium" (dict
"requests" (dict "cpu" "500m" "memory" "1024Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "750m" "memory" "1536Mi" "ephemeral-storage" "2Gi")
)
"large" (dict
"requests" (dict "cpu" "1.0" "memory" "2048Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "1.5" "memory" "3072Mi" "ephemeral-storage" "2Gi")
)
"xlarge" (dict
"requests" (dict "cpu" "1.0" "memory" "3072Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "3.0" "memory" "6144Mi" "ephemeral-storage" "2Gi")
)
"2xlarge" (dict
"requests" (dict "cpu" "1.0" "memory" "3072Mi" "ephemeral-storage" "50Mi")
"limits" (dict "cpu" "6.0" "memory" "12288Mi" "ephemeral-storage" "2Gi")
)
}}
{{- if hasKey $presets .type -}}
{{- index $presets .type | toYaml -}}
{{- else -}}
{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_secrets.tpl
Executable
+185
View File
@@ -0,0 +1,185 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Generate secret name.
Usage:
{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }}
Params:
- existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user
to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility.
+info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret
- defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment.
- context - Dict - Required. The context for the template evaluation.
*/}}
{{- define "common.secrets.name" -}}
{{- $name := (include "common.names.fullname" .context) -}}
{{- if .defaultNameSuffix -}}
{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- with .existingSecret -}}
{{- if not (typeIs "string" .) -}}
{{- with .name -}}
{{- $name = . -}}
{{- end -}}
{{- else -}}
{{- $name = . -}}
{{- end -}}
{{- end -}}
{{- printf "%s" $name -}}
{{- end -}}
{{/*
Generate secret key.
Usage:
{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }}
Params:
- existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user
to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility.
+info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret
- key - String - Required. Name of the key in the secret.
*/}}
{{- define "common.secrets.key" -}}
{{- $key := .key -}}
{{- if .existingSecret -}}
{{- if not (typeIs "string" .existingSecret) -}}
{{- if .existingSecret.keyMapping -}}
{{- $key = index .existingSecret.keyMapping $.key -}}
{{- end -}}
{{- end }}
{{- end -}}
{{- printf "%s" $key -}}
{{- end -}}
{{/*
Generate secret password or retrieve one if already created.
Usage:
{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }}
Params:
- secret - String - Required - Name of the 'Secret' resource where the password is stored.
- key - String - Required - Name of the key in the secret.
- providedValues - List<String> - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value.
- length - int - Optional - Length of the generated random password.
- strong - Boolean - Optional - Whether to add symbols to the generated random password.
- chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart.
- context - Context - Required - Parent context.
- failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets.
- skipB64enc - Boolean - Optional - Default to false. If set to true, no the secret will not be base64 encrypted.
- skipQuote - Boolean - Optional - Default to false. If set to true, no quotes will be added around the secret.
The order in which this function returns a secret password:
1. Already existing 'Secret' resource
(If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned)
2. Password provided via the values.yaml
(If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned)
3. Randomly generated secret password
(A new random secret password with the length specified in the 'length' parameter will be generated and returned)
*/}}
{{- define "common.secrets.passwords.manage" -}}
{{- $password := "" }}
{{- $subchart := "" }}
{{- $chartName := default "" .chartName }}
{{- $passwordLength := default 10 .length }}
{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }}
{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }}
{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }}
{{- if $secretData }}
{{- if hasKey $secretData .key }}
{{- $password = index $secretData .key | b64dec }}
{{- else if not (eq .failOnNew false) }}
{{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}}
{{- end -}}
{{- end }}
{{- if not $password }}
{{- if $providedPasswordValue }}
{{- $password = $providedPasswordValue | toString }}
{{- else }}
{{- if .context.Values.enabled }}
{{- $subchart = $chartName }}
{{- end -}}
{{- if not (eq .failOnNew false) }}
{{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}}
{{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}}
{{- $passwordValidationErrors := list $requiredPasswordError -}}
{{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}}
{{- end }}
{{- if .strong }}
{{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }}
{{- $password = randAscii $passwordLength }}
{{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }}
{{- $password = printf "%s%s" $subStr $password | toString | shuffle }}
{{- else }}
{{- $password = randAlphaNum $passwordLength }}
{{- end }}
{{- end -}}
{{- end -}}
{{- if not .skipB64enc }}
{{- $password = $password | b64enc }}
{{- end -}}
{{- if .skipQuote -}}
{{- printf "%s" $password -}}
{{- else -}}
{{- printf "%s" $password | quote -}}
{{- end -}}
{{- end -}}
{{/*
Reuses the value from an existing secret, otherwise sets its value to a default value.
Usage:
{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }}
Params:
- secret - String - Required - Name of the 'Secret' resource where the password is stored.
- key - String - Required - Name of the key in the secret.
- defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value.
- context - Context - Required - Parent context.
*/}}
{{- define "common.secrets.lookup" -}}
{{- $value := "" -}}
{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}}
{{- if and $secretData (hasKey $secretData .key) -}}
{{- $value = index $secretData .key -}}
{{- else if .defaultValue -}}
{{- $value = .defaultValue | toString | b64enc -}}
{{- end -}}
{{- if $value -}}
{{- printf "%s" $value -}}
{{- end -}}
{{- end -}}
{{/*
Returns whether a previous generated secret already exists
Usage:
{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }}
Params:
- secret - String - Required - Name of the 'Secret' resource where the password is stored.
- context - Context - Required - Parent context.
*/}}
{{- define "common.secrets.exists" -}}
{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }}
{{- if $secret }}
{{- true -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_storage.tpl
Executable
+21
View File
@@ -0,0 +1,21 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Return the proper Storage Class
{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }}
*/}}
{{- define "common.storage.class" -}}
{{- $storageClass := (.global).storageClass | default .persistence.storageClass | default (.global).defaultStorageClass | default "" -}}
{{- if $storageClass -}}
{{- if (eq "-" $storageClass) -}}
{{- printf "storageClassName: \"\"" -}}
{{- else -}}
{{- printf "storageClassName: %s" $storageClass -}}
{{- end -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_tplvalues.tpl
Executable
+38
View File
@@ -0,0 +1,38 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Renders a value that contains template perhaps with scope if the scope is present.
Usage:
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
*/}}
{{- define "common.tplvalues.render" -}}
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
{{- if contains "{{" (toJson .value) }}
{{- if .scope }}
{{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
{{- else }}
{{- tpl $value .context }}
{{- end }}
{{- else }}
{{- $value }}
{{- end }}
{{- end -}}
{{/*
Merge a list of values that contains template after rendering them.
Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge
Usage:
{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }}
*/}}
{{- define "common.tplvalues.merge" -}}
{{- $dst := dict -}}
{{- range .values -}}
{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}}
{{- end -}}
{{ $dst | toYaml }}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_utils.tpl
Executable
+77
View File
@@ -0,0 +1,77 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Print instructions to get a secret value.
Usage:
{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }}
*/}}
{{- define "common.utils.secret.getvalue" -}}
{{- $varname := include "common.utils.fieldToEnvVar" . -}}
export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d)
{{- end -}}
{{/*
Build env var name given a field
Usage:
{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }}
*/}}
{{- define "common.utils.fieldToEnvVar" -}}
{{- $fieldNameSplit := splitList "-" .field -}}
{{- $upperCaseFieldNameSplit := list -}}
{{- range $fieldNameSplit -}}
{{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}}
{{- end -}}
{{ join "_" $upperCaseFieldNameSplit }}
{{- end -}}
{{/*
Gets a value from .Values given
Usage:
{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }}
*/}}
{{- define "common.utils.getValueFromKey" -}}
{{- $splitKey := splitList "." .key -}}
{{- $value := "" -}}
{{- $latestObj := $.context.Values -}}
{{- range $splitKey -}}
{{- if not $latestObj -}}
{{- printf "please review the entire path of '%s' exists in values" $.key | fail -}}
{{- end -}}
{{- $value = ( index $latestObj . ) -}}
{{- $latestObj = $value -}}
{{- end -}}
{{- printf "%v" (default "" $value) -}}
{{- end -}}
{{/*
Returns first .Values key with a defined value or first of the list if all non-defined
Usage:
{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }}
*/}}
{{- define "common.utils.getKeyFromList" -}}
{{- $key := first .keys -}}
{{- $reverseKeys := reverse .keys }}
{{- range $reverseKeys }}
{{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }}
{{- if $value -}}
{{- $key = . }}
{{- end -}}
{{- end -}}
{{- printf "%s" $key -}}
{{- end -}}
{{/*
Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376).
Usage:
{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }}
*/}}
{{- define "common.utils.checksumTemplate" -}}
{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}}
{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/_warnings.tpl
Executable
+109
View File
@@ -0,0 +1,109 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Warning about using rolling tag.
Usage:
{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }}
*/}}
{{- define "common.warnings.rollingTag" -}}
{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }}
WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment.
+info https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html
{{- end }}
{{- end -}}
{{/*
Warning about replaced images from the original.
Usage:
{{ include "common.warnings.modifiedImages" (dict "images" (list .Values.path.to.the.imageRoot) "context" $) }}
*/}}
{{- define "common.warnings.modifiedImages" -}}
{{- $affectedImages := list -}}
{{- $printMessage := false -}}
{{- $originalImages := .context.Chart.Annotations.images -}}
{{- range .images -}}
{{- $fullImageName := printf (printf "%s/%s:%s" .registry .repository .tag) -}}
{{- if not (contains $fullImageName $originalImages) }}
{{- $affectedImages = append $affectedImages (printf "%s/%s:%s" .registry .repository .tag) -}}
{{- $printMessage = true -}}
{{- end -}}
{{- end -}}
{{- if $printMessage }}
⚠ SECURITY WARNING: Original containers have been substituted. This Helm chart was designed, tested, and validated on multiple platforms using a specific set of Bitnami and Tanzu Application Catalog containers. Substituting other containers is likely to cause degraded security and performance, broken chart features, and missing environment variables.
Substituted images detected:
{{- range $affectedImages }}
- {{ . }}
{{- end }}
{{- end -}}
{{- end -}}
{{/*
Warning about not setting the resource object in all deployments.
Usage:
{{ include "common.warnings.resources" (dict "sections" (list "path1" "path2") context $) }}
Example:
{{- include "common.warnings.resources" (dict "sections" (list "csiProvider.provider" "server" "volumePermissions" "") "context" $) }}
The list in the example assumes that the following values exist:
- csiProvider.provider.resources
- server.resources
- volumePermissions.resources
- resources
*/}}
{{- define "common.warnings.resources" -}}
{{- $values := .context.Values -}}
{{- $printMessage := false -}}
{{ $affectedSections := list -}}
{{- range .sections -}}
{{- if eq . "" -}}
{{/* Case where the resources section is at the root (one main deployment in the chart) */}}
{{- if not (index $values "resources") -}}
{{- $affectedSections = append $affectedSections "resources" -}}
{{- $printMessage = true -}}
{{- end -}}
{{- else -}}
{{/* Case where the are multiple resources sections (more than one main deployment in the chart) */}}
{{- $keys := split "." . -}}
{{/* We iterate through the different levels until arriving to the resource section. Example: a.b.c.resources */}}
{{- $section := $values -}}
{{- range $keys -}}
{{- $section = index $section . -}}
{{- end -}}
{{- if not (index $section "resources") -}}
{{/* If the section has enabled=false or replicaCount=0, do not include it */}}
{{- if and (hasKey $section "enabled") -}}
{{- if index $section "enabled" -}}
{{/* enabled=true */}}
{{- $affectedSections = append $affectedSections (printf "%s.resources" .) -}}
{{- $printMessage = true -}}
{{- end -}}
{{- else if and (hasKey $section "replicaCount") -}}
{{/* We need a casting to int because number 0 is not treated as an int by default */}}
{{- if (gt (index $section "replicaCount" | int) 0) -}}
{{/* replicaCount > 0 */}}
{{- $affectedSections = append $affectedSections (printf "%s.resources" .) -}}
{{- $printMessage = true -}}
{{- end -}}
{{- else -}}
{{/* Default case, add it to the affected sections */}}
{{- $affectedSections = append $affectedSections (printf "%s.resources" .) -}}
{{- $printMessage = true -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- if $printMessage }}
WARNING: There are "resources" sections in the chart not set. Using "resourcesPreset" is not recommended for production. For production installations, please set the following values according to your workload needs:
{{- range $affectedSections }}
- {{ . }}
{{- end }}
+info https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_cassandra.tpl
Executable
+77
View File
@@ -0,0 +1,77 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate Cassandra required passwords are not empty.
Usage:
{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret"
- subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.cassandra.passwords" -}}
{{- $existingSecret := include "common.cassandra.values.existingSecret" . -}}
{{- $enabled := include "common.cassandra.values.enabled" . -}}
{{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}}
{{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPassword -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for existingSecret.
Usage:
{{ include "common.cassandra.values.existingSecret" (dict "context" $) }}
Params:
- subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false
*/}}
{{- define "common.cassandra.values.existingSecret" -}}
{{- if .subchart -}}
{{- .context.Values.cassandra.dbUser.existingSecret | quote -}}
{{- else -}}
{{- .context.Values.dbUser.existingSecret | quote -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled cassandra.
Usage:
{{ include "common.cassandra.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.cassandra.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.cassandra.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key dbUser
Usage:
{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false
*/}}
{{- define "common.cassandra.values.key.dbUser" -}}
{{- if .subchart -}}
cassandra.dbUser
{{- else -}}
dbUser
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_mariadb.tpl
Executable
+108
View File
@@ -0,0 +1,108 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate MariaDB required passwords are not empty.
Usage:
{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret"
- subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.mariadb.passwords" -}}
{{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}}
{{- $enabled := include "common.mariadb.values.enabled" . -}}
{{- $architecture := include "common.mariadb.values.architecture" . -}}
{{- $authPrefix := include "common.mariadb.values.key.auth" . -}}
{{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}}
{{- $valueKeyUsername := printf "%s.username" $authPrefix -}}
{{- $valueKeyPassword := printf "%s.password" $authPrefix -}}
{{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}}
{{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }}
{{- if not (empty $valueUsername) -}}
{{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPassword -}}
{{- end -}}
{{- if (eq $architecture "replication") -}}
{{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}}
{{- end -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for existingSecret.
Usage:
{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false
*/}}
{{- define "common.mariadb.values.auth.existingSecret" -}}
{{- if .subchart -}}
{{- .context.Values.mariadb.auth.existingSecret | quote -}}
{{- else -}}
{{- .context.Values.auth.existingSecret | quote -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled mariadb.
Usage:
{{ include "common.mariadb.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.mariadb.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.mariadb.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for architecture
Usage:
{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false
*/}}
{{- define "common.mariadb.values.architecture" -}}
{{- if .subchart -}}
{{- .context.Values.mariadb.architecture -}}
{{- else -}}
{{- .context.Values.architecture -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key auth
Usage:
{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false
*/}}
{{- define "common.mariadb.values.key.auth" -}}
{{- if .subchart -}}
mariadb.auth
{{- else -}}
auth
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_mongodb.tpl
Executable
+113
View File
@@ -0,0 +1,113 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate MongoDB&reg; required passwords are not empty.
Usage:
{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where MongoDB&reg; values are stored, e.g: "mongodb-passwords-secret"
- subchart - Boolean - Optional. Whether MongoDB&reg; is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.mongodb.passwords" -}}
{{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}}
{{- $enabled := include "common.mongodb.values.enabled" . -}}
{{- $authPrefix := include "common.mongodb.values.key.auth" . -}}
{{- $architecture := include "common.mongodb.values.architecture" . -}}
{{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}}
{{- $valueKeyUsername := printf "%s.username" $authPrefix -}}
{{- $valueKeyDatabase := printf "%s.database" $authPrefix -}}
{{- $valueKeyPassword := printf "%s.password" $authPrefix -}}
{{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}}
{{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}}
{{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}}
{{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }}
{{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }}
{{- if and $valueUsername $valueDatabase -}}
{{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPassword -}}
{{- end -}}
{{- if (eq $architecture "replicaset") -}}
{{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}}
{{- end -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for existingSecret.
Usage:
{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false
*/}}
{{- define "common.mongodb.values.auth.existingSecret" -}}
{{- if .subchart -}}
{{- .context.Values.mongodb.auth.existingSecret | quote -}}
{{- else -}}
{{- .context.Values.auth.existingSecret | quote -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled mongodb.
Usage:
{{ include "common.mongodb.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.mongodb.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.mongodb.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key auth
Usage:
{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MongoDB&reg; is used as subchart or not. Default: false
*/}}
{{- define "common.mongodb.values.key.auth" -}}
{{- if .subchart -}}
mongodb.auth
{{- else -}}
auth
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for architecture
Usage:
{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MongoDB&reg; is used as subchart or not. Default: false
*/}}
{{- define "common.mongodb.values.architecture" -}}
{{- if .subchart -}}
{{- .context.Values.mongodb.architecture -}}
{{- else -}}
{{- .context.Values.architecture -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_mysql.tpl
Executable
+108
View File
@@ -0,0 +1,108 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate MySQL required passwords are not empty.
Usage:
{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret"
- subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.mysql.passwords" -}}
{{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}}
{{- $enabled := include "common.mysql.values.enabled" . -}}
{{- $architecture := include "common.mysql.values.architecture" . -}}
{{- $authPrefix := include "common.mysql.values.key.auth" . -}}
{{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}}
{{- $valueKeyUsername := printf "%s.username" $authPrefix -}}
{{- $valueKeyPassword := printf "%s.password" $authPrefix -}}
{{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}}
{{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }}
{{- if not (empty $valueUsername) -}}
{{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPassword -}}
{{- end -}}
{{- if (eq $architecture "replication") -}}
{{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}}
{{- end -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for existingSecret.
Usage:
{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false
*/}}
{{- define "common.mysql.values.auth.existingSecret" -}}
{{- if .subchart -}}
{{- .context.Values.mysql.auth.existingSecret | quote -}}
{{- else -}}
{{- .context.Values.auth.existingSecret | quote -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled mysql.
Usage:
{{ include "common.mysql.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.mysql.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.mysql.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for architecture
Usage:
{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false
*/}}
{{- define "common.mysql.values.architecture" -}}
{{- if .subchart -}}
{{- .context.Values.mysql.architecture -}}
{{- else -}}
{{- .context.Values.architecture -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key auth
Usage:
{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false
*/}}
{{- define "common.mysql.values.key.auth" -}}
{{- if .subchart -}}
mysql.auth
{{- else -}}
auth
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_postgresql.tpl
Executable
+134
View File
@@ -0,0 +1,134 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate PostgreSQL required passwords are not empty.
Usage:
{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret"
- subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.postgresql.passwords" -}}
{{- $existingSecret := include "common.postgresql.values.existingSecret" . -}}
{{- $enabled := include "common.postgresql.values.enabled" . -}}
{{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}}
{{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}}
{{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}}
{{- if (eq $enabledReplication "true") -}}
{{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}}
{{- end -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to decide whether evaluate global values.
Usage:
{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }}
Params:
- key - String - Required. Field to be evaluated within global, e.g: "existingSecret"
*/}}
{{- define "common.postgresql.values.use.global" -}}
{{- if .context.Values.global -}}
{{- if .context.Values.global.postgresql -}}
{{- index .context.Values.global.postgresql .key | quote -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for existingSecret.
Usage:
{{ include "common.postgresql.values.existingSecret" (dict "context" $) }}
*/}}
{{- define "common.postgresql.values.existingSecret" -}}
{{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}}
{{- if .subchart -}}
{{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}}
{{- else -}}
{{- default (.context.Values.existingSecret | quote) $globalValue -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled postgresql.
Usage:
{{ include "common.postgresql.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.postgresql.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.postgresql.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key postgressPassword.
Usage:
{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false
*/}}
{{- define "common.postgresql.values.key.postgressPassword" -}}
{{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}}
{{- if not $globalValue -}}
{{- if .subchart -}}
postgresql.postgresqlPassword
{{- else -}}
postgresqlPassword
{{- end -}}
{{- else -}}
global.postgresql.postgresqlPassword
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled.replication.
Usage:
{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false
*/}}
{{- define "common.postgresql.values.enabled.replication" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.postgresql.replication.enabled -}}
{{- else -}}
{{- printf "%v" .context.Values.replication.enabled -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for the key replication.password.
Usage:
{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false
*/}}
{{- define "common.postgresql.values.key.replicationPassword" -}}
{{- if .subchart -}}
postgresql.replication.password
{{- else -}}
replication.password
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_redis.tpl
Executable
+81
View File
@@ -0,0 +1,81 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate Redis&reg; required passwords are not empty.
Usage:
{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }}
Params:
- secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret"
- subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false
*/}}
{{- define "common.validations.values.redis.passwords" -}}
{{- $enabled := include "common.redis.values.enabled" . -}}
{{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}}
{{- $standarizedVersion := include "common.redis.values.standarized.version" . }}
{{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }}
{{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }}
{{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }}
{{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }}
{{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}}
{{- $requiredPasswords := list -}}
{{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}}
{{- if eq $useAuth "true" -}}
{{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}}
{{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}}
{{- end -}}
{{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right value for enabled redis.
Usage:
{{ include "common.redis.values.enabled" (dict "context" $) }}
*/}}
{{- define "common.redis.values.enabled" -}}
{{- if .subchart -}}
{{- printf "%v" .context.Values.redis.enabled -}}
{{- else -}}
{{- printf "%v" (not .context.Values.enabled) -}}
{{- end -}}
{{- end -}}
{{/*
Auxiliary function to get the right prefix path for the values
Usage:
{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }}
Params:
- subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false
*/}}
{{- define "common.redis.values.keys.prefix" -}}
{{- if .subchart -}}redis.{{- else -}}{{- end -}}
{{- end -}}
{{/*
Checks whether the redis chart's includes the standarizations (version >= 14)
Usage:
{{ include "common.redis.values.standarized.version" (dict "context" $) }}
*/}}
{{- define "common.redis.values.standarized.version" -}}
{{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}}
{{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }}
{{- if $standarizedAuthValues -}}
{{- true -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/templates/validations/_validations.tpl
Executable
+51
View File
@@ -0,0 +1,51 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{/* vim: set filetype=mustache: */}}
{{/*
Validate values must not be empty.
Usage:
{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}}
{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}}
{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }}
Validate value params:
- valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password"
- secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret"
- field - String - Optional. Name of the field in the secret data, e.g: "mysql-password"
*/}}
{{- define "common.validations.values.multiple.empty" -}}
{{- range .required -}}
{{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}}
{{- end -}}
{{- end -}}
{{/*
Validate a value must not be empty.
Usage:
{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }}
Validate value params:
- valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password"
- secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret"
- field - String - Optional. Name of the field in the secret data, e.g: "mysql-password"
- subchart - String - Optional - Name of the subchart that the validated password is part of.
*/}}
{{- define "common.validations.values.single.empty" -}}
{{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }}
{{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }}
{{- if not $value -}}
{{- $varname := "my-value" -}}
{{- $getCurrentValue := "" -}}
{{- if and .secret .field -}}
{{- $varname = include "common.utils.fieldToEnvVar" . -}}
{{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}}
{{- end -}}
{{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}}
{{- end -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/charts/common/values.yaml
Executable
+8
View File
@@ -0,0 +1,8 @@
# Copyright Broadcom, Inc. All Rights Reserved.
# SPDX-License-Identifier: APACHE-2.0
## bitnami/common
## It is required by CI/CD tools and processes.
## @skip exampleValue
##
exampleValue: common-chart
spark-cluster/base/charts/spark-9.2.10/spark/templates/NOTES.txt
Executable
+104
View File
@@ -0,0 +1,104 @@
CHART NAME: {{ .Chart.Name }}
CHART VERSION: {{ .Chart.Version }}
APP VERSION: {{ .Chart.AppVersion }}
** Please be patient while the chart is being deployed **
{{- if .Values.diagnosticMode.enabled }}
The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with:
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }}
Get the list of pods by executing:
kubectl get pods --namespace {{ include "common.names.namespace" . }} -l app.kubernetes.io/instance={{ .Release.Name }}
Access the pod you want to debug by executing
kubectl exec --namespace {{ include "common.names.namespace" . }} -ti <NAME OF THE POD> -- bash
In order to replicate the container startup scripts execute this command:
/opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh
{{- else }}
1. Get the Spark master WebUI URL by running these commands:
{{- if .Values.ingress.enabled }}
export HOSTNAME=$(kubectl get ingress --namespace {{ include "common.names.namespace" . }} {{ printf "%s-ingress" (include "common.names.fullname" .) }} -o jsonpath='{.spec.rules[0].host}')
echo "Spark-master URL: http://$HOSTNAME/"
{{- else -}}
{{- if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.spec.ports[?(@.name=='http')].nodePort}" services {{ printf "%s-master-svc" (include "common.names.fullname" .) }})
export NODE_IP=$(kubectl get nodes --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ include "common.names.namespace" . }} svc -w {{ printf "%s-master-svc" (include "common.names.fullname" .) }}'
export SERVICE_IP=$(kubectl get --namespace {{ include "common.names.namespace" . }} svc {{ printf "%s-master-svc" (include "common.names.fullname" .) }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }")
echo http://$SERVICE_IP:{{ .Values.service.ports.http }}
{{- else if contains "ClusterIP" .Values.service.type }}
kubectl port-forward --namespace {{ include "common.names.namespace" . }} svc/{{ printf "%s-master-svc" (include "common.names.fullname" .) }} {{ default "80" .Values.service.ports.http }}:{{ default "80" .Values.service.ports.http }}
echo "Visit http://127.0.0.1:{{ .Values.service.ports.http }} to use your application"
{{- end }}
{{- end }}
2. Submit an application to the cluster:
To submit an application to the cluster the spark-submit script must be used. That script can be
obtained at https://github.com/apache/spark/tree/master/bin. Also you can use kubectl run.
{{- if or (eq "NodePort" .Values.service.type) (eq "LoadBalancer" .Values.service.type) }}
Run the commands below to obtain the master IP and submit your application.
{{- end }}
export EXAMPLE_JAR=$(kubectl exec -ti --namespace {{ include "common.names.namespace" . }} {{ printf "%s-worker-0" (include "common.names.fullname" .) }} -- find examples/jars/ -name 'spark-example*\.jar' | tr -d '\r')
{{- if eq "NodePort" .Values.service.type }}
export SUBMIT_PORT=$(kubectl get --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.spec.ports[?(@.name=='cluster')].nodePort}" services {{ printf "%s-master-svc" (include "common.names.fullname" .) }})
export SUBMIT_IP=$(kubectl get nodes --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
kubectl run --namespace {{ include "common.names.namespace" . }} {{ printf "%s-client" (include "common.names.fullname" .) }} --rm --tty -i --restart='Never' \
--image {{ template "spark.image" . }} \
-- spark-submit --master spark://$SUBMIT_IP:$SUBMIT_PORT \
--class org.apache.spark.examples.SparkPi \
--deploy-mode cluster \
$EXAMPLE_JAR 1000
{{- else if eq "LoadBalancer" .Values.service.type }}
export SUBMIT_IP=$(kubectl get --namespace {{ include "common.names.namespace" . }} svc {{ printf "%s-master-svc" (include "common.names.fullname" .) }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }")
kubectl run --namespace {{ include "common.names.namespace" . }} {{ printf "%s-client" (include "common.names.fullname" .) }} --rm --tty -i --restart='Never' \
--image {{ template "spark.image" . }} \
-- spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.ports.cluster }} \
--deploy-mode cluster \
--class org.apache.spark.examples.SparkPi \
$EXAMPLE_JAR 1000
{{- else }}
kubectl exec -ti --namespace {{ include "common.names.namespace" . }} {{ printf "%s-worker-0" (include "common.names.fullname" .) }} -- spark-submit --master spark://{{ printf "%s-master-svc" (include "common.names.fullname" .) }}:{{ .Values.service.ports.cluster }} \
--class org.apache.spark.examples.SparkPi \
$EXAMPLE_JAR 5
** IMPORTANT: When submit an application from outside the cluster service type should be set to the NodePort or LoadBalancer. **
{{- end }}
** IMPORTANT: When submit an application the --master parameter should be set to the service IP, if not, the application will not resolve the master. **
{{- end }}
{{ include "spark.checkRollingTags" . }}
{{ include "spark.validateValues" . }}
{{- include "common.warnings.resources" (dict "sections" (list "master" "security.ssl" "worker") "context" $) }}
{{- include "common.warnings.modifiedImages" (dict "images" (list .Values.image) "context" $) }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/_helpers.tpl
Executable
+132
View File
@@ -0,0 +1,132 @@
{{/*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- /* vim: set filetype=mustache: */}}
{{/*
Return the proper Spark image name
*/}}
{{- define "spark.image" -}}
{{- include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) -}}
{{- end -}}
{{/*
Return the proper Docker Image Registry Secret Names
*/}}
{{- define "spark.imagePullSecrets" -}}
{{- include "common.images.pullSecrets" (dict "images" (list .Values.image) "global" .Values.global) -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "spark.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "common.names.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/* Get the secret for passwords */}}
{{- define "spark.passwordsSecretName" -}}
{{- if .Values.security.passwordsSecretName -}}
{{- printf "%s" .Values.security.passwordsSecretName -}}
{{- else }}
{{- printf "%s-secret" (include "common.names.fullname" .) -}}
{{- end }}
{{- end -}}
{{/*
Return the secret containing Spark TLS certificates
*/}}
{{- define "spark.tlsSecretName" -}}
{{- $secretName := coalesce .Values.security.ssl.existingSecret .Values.security.certificatesSecretName -}}
{{- if $secretName -}}
{{- printf "%s" (tpl $secretName $) -}}
{{- else -}}
{{- printf "%s-crt" (include "common.names.fullname" .) -}}
{{- end -}}
{{- end -}}
{{/*
Return true if a TLS secret object should be created
*/}}
{{- define "spark.createTlsSecret" -}}
{{- if and .Values.security.ssl.autoGenerated .Values.security.ssl.enabled (not .Values.security.ssl.existingSecret) (not .Values.security.certificatesSecretName) }}
{{- true -}}
{{- end -}}
{{- end -}}
{{/* Check if there are rolling tags in the images */}}
{{- define "spark.checkRollingTags" -}}
{{- include "common.warnings.rollingTag" .Values.image -}}
{{- end -}}
{{/* Validate values of Spark - Incorrect extra volume settings */}}
{{- define "spark.validateValues.extraVolumes" -}}
{{- if and (.Values.worker.extraVolumes) (not .Values.worker.extraVolumeMounts) -}}
spark: missing-worker-extra-volume-mounts
You specified worker extra volumes but no mount points for them. Please set
the extraVolumeMounts value
{{- end -}}
{{- end -}}
{/* Validate values of Spark - number of workers must be greater than 0 */}}
{{- define "spark.validateValues.workerCount" -}}
{{- $replicaCount := int .Values.worker.replicaCount }}
{{- if lt $replicaCount 1 -}}
spark: workerCount
Worker replicas must be greater than 0!!
Please set a valid worker count size (--set worker.replicaCount=X)
{{- end -}}
{{- end -}}
{{/* Validate values of Spark - Security SSL enabled */}}
{{- define "spark.validateValues.security.ssl" -}}
{{- if and .Values.security.ssl.enabled (not .Values.security.ssl.autoGenerated) (not .Values.security.ssl.existingSecret) (not .Values.security.certificatesSecretName) }}
spark: security.ssl.enabled
In order to enable Security SSL, you also need to provide
an existing secret containing the Keystore and Truststore or
enable auto-generated certificates.
{{- end -}}
{{- end -}}
{{/*
Compile all warnings into a single message, and call fail.
*/}}
{{- define "spark.validateValues" -}}
{{- $messages := list -}}
{{- $messages := append $messages (include "spark.validateValues.extraVolumes" .) -}}
{{- $messages := append $messages (include "spark.validateValues.workerCount" .) -}}
{{- $messages := append $messages (include "spark.validateValues.security.ssl" .) -}}
{{- $messages := without $messages "" -}}
{{- $message := join "\n" $messages -}}
{{- if $message -}}
{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}}
{{- end -}}
{{- end -}}
{{/*
Get the initialization scripts volume name.
*/}}
{{- define "spark.initScripts" -}}
{{- printf "%s-init-scripts" (include "common.names.fullname" .) -}}
{{- end -}}
{{/*
Get the initialization scripts ConfigMap name.
*/}}
{{- define "spark.initScriptsCM" -}}
{{- printf "%s" .Values.initScriptsCM -}}
{{- end -}}
{{/*
Get the initialization scripts Secret name.
*/}}
{{- define "spark.initScriptsSecret" -}}
{{- printf "%s" .Values.initScriptsSecret -}}
{{- end -}}
spark-cluster/base/charts/spark-9.2.10/spark/templates/extra-list.yaml
Executable
+9
View File
@@ -0,0 +1,9 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- range .Values.extraDeploy }}
---
{{ include "common.tplvalues.render" (dict "value" . "context" $) }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/headless-svc.yaml
Executable
+19
View File
@@ -0,0 +1,19 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
apiVersion: v1
kind: Service
metadata:
name: {{ printf "%s-headless" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if or .Values.service.headless.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.service.headless.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
clusterIP: None
selector: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/hpa-worker.yaml
Executable
+49
View File
@@ -0,0 +1,49 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.worker.autoscaling.enabled }}
apiVersion: {{ include "common.capabilities.hpa.apiVersion" ( dict "context" $ ) }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ printf "%s-autoscaler" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: worker-autoscaler
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }}
kind: statefulset
name: {{ printf "%s-worker" (include "common.names.fullname" .) }}
minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
metrics:
{{- if .Values.worker.autoscaling.targetMemory }}
- type: Resource
resource:
name: memory
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.worker.autoscaling.targetMemory }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.worker.autoscaling.targetMemory }}
{{- end }}
{{- end }}
{{- if .Values.worker.autoscaling.targetCPU }}
- type: Resource
resource:
name: cpu
{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
targetAverageUtilization: {{ .Values.worker.autoscaling.targetCPU }}
{{- else }}
target:
type: Utilization
averageUtilization: {{ .Values.worker.autoscaling.targetCPU }}
{{- end }}
{{- end }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/ingress.yaml
Executable
+60
View File
@@ -0,0 +1,60 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.ingress.enabled }}
apiVersion: {{ template "common.capabilities.ingress.apiVersion" . }}
kind: Ingress
metadata:
name: {{ printf "%s-ingress" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: ingress
{{- if or .Values.ingress.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.ingress.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.ingressClassName (eq "true" (include "common.ingress.supportsIngressClassname" .)) }}
ingressClassName: {{ .Values.ingress.ingressClassName | quote }}
{{- end }}
rules:
{{- if .Values.ingress.hostname }}
- host: {{ .Values.ingress.hostname }}
http:
paths:
{{- if .Values.ingress.extraPaths }}
{{- toYaml .Values.ingress.extraPaths | nindent 10 }}
{{- end }}
- path: {{ .Values.ingress.path }}
{{- if eq "true" (include "common.ingress.supportsPathType" .) }}
pathType: {{ .Values.ingress.pathType }}
{{- end }}
backend: {{- include "common.ingress.backend" (dict "serviceName" (printf "%s-master-svc" (include "common.names.fullname" .)) "servicePort" "http" "context" $) | nindent 14 }}
{{- end }}
{{- range .Values.ingress.extraHosts }}
- host: {{ .name | quote }}
http:
paths:
- path: {{ default "/" .path }}
{{- if eq "true" (include "common.ingress.supportsPathType" $) }}
pathType: {{ default "ImplementationSpecific" .pathType }}
{{- end }}
backend: {{- include "common.ingress.backend" (dict "serviceName" (printf "%s-master-svc" (include "common.names.fullname" $)) "servicePort" "http" "context" $) | nindent 14 }}
{{- end }}
{{- if .Values.ingress.extraRules }}
{{- include "common.tplvalues.render" (dict "value" .Values.ingress.extraRules "context" $) | nindent 4 }}
{{- end }}
{{- if or (and .Values.ingress.tls (or (include "common.ingress.certManagerRequest" ( dict "annotations" .Values.ingress.annotations )) .Values.ingress.selfSigned)) .Values.ingress.extraTls }}
tls:
{{- if and .Values.ingress.tls (or (include "common.ingress.certManagerRequest" ( dict "annotations" .Values.ingress.annotations )) .Values.ingress.selfSigned) }}
- hosts:
- {{ .Values.ingress.hostname }}
secretName: {{ printf "%s-tls" .Values.ingress.hostname }}
{{- end }}
{{- if .Values.ingress.extraTls }}
{{- include "common.tplvalues.render" ( dict "value" .Values.ingress.extraTls "context" $ ) | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/init-configmap.yaml
Executable
+18
View File
@@ -0,0 +1,18 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.initScripts }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ printf "%s-init-scripts" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
data:
{{- include "common.tplvalues.render" ( dict "value" .Values.initScripts "context" $ ) | nindent 2 }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/networkpolicy-master.yaml
Executable
+91
View File
@@ -0,0 +1,91 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.master.networkPolicy.enabled }}
kind: NetworkPolicy
apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }}
metadata:
name: {{ printf "%s-master" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: master
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: master
policyTypes:
- Ingress
- Egress
{{- if .Values.master.networkPolicy.allowExternalEgress }}
egress:
- {}
{{- else }}
egress:
# Allow dns resolution
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Allow outbound connections to other worker pods
- ports:
- port: {{ .Values.worker.containerPorts.cluster }}
- port: {{ ternary .Values.worker.containerPorts.https .Values.worker.containerPorts.http .Values.security.ssl.enabled }}
to:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }}
app.kubernetes.io/component: worker
# Allow outbound connections to other master pods
- ports:
- port: {{ .Values.service.ports.cluster }}
- port: {{ ternary .Values.service.ports.https .Values.service.ports.http .Values.security.ssl.enabled }}
- port: {{ .Values.master.containerPorts.cluster }}
- port: {{ ternary .Values.master.containerPorts.https .Values.master.containerPorts.http .Values.security.ssl.enabled }}
to:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }}
app.kubernetes.io/component: master
{{- if .Values.master.networkPolicy.extraEgress }}
{{- include "common.tplvalues.render" ( dict "value" .Values.master.networkPolicy.extraEgress "context" $ ) | nindent 4 }}
{{- end }}
{{- end }}
ingress:
- ports:
- port: {{ .Values.service.ports.cluster }}
- port: {{ ternary .Values.service.ports.https .Values.service.ports.http .Values.security.ssl.enabled }}
- port: {{ .Values.master.containerPorts.cluster }}
- port: {{ ternary .Values.master.containerPorts.https .Values.master.containerPorts.http .Values.security.ssl.enabled }}
{{- if not .Values.master.networkPolicy.allowExternal }}
from:
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 14 }}
- podSelector:
matchLabels:
{{ template "common.names.fullname" . }}-master: "true"
{{- if .Values.master.networkPolicy.ingressNSMatchLabels }}
- namespaceSelector:
matchLabels:
{{- range $key, $value := .Values.master.networkPolicy.ingressNSMatchLabels }}
{{ $key | quote }}: {{ $value | quote }}
{{- end }}
{{- if .Values.master.networkPolicy.ingressNSPodMatchLabels }}
podSelector:
matchLabels:
{{- range $key, $value := .Values.master.networkPolicy.ingressNSPodMatchLabels }}
{{ $key | quote }}: {{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.master.networkPolicy.extraIngress }}
{{- include "common.tplvalues.render" ( dict "value" .Values.master.networkPolicy.extraIngress "context" $ ) | nindent 4 }}
{{- end }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/networkpolicy-worker.yaml
Executable
+89
View File
@@ -0,0 +1,89 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.worker.networkPolicy.enabled }}
kind: NetworkPolicy
apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }}
metadata:
name: {{ printf "%s-worker" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: worker
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: worker
policyTypes:
- Ingress
- Egress
{{- if .Values.worker.networkPolicy.allowExternalEgress }}
egress:
- {}
{{- else }}
egress:
# Allow dns resolution
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Allow outbound connections to other worker pods
- ports:
- port: {{ .Values.worker.containerPorts.cluster }}
- port: {{ ternary .Values.worker.containerPorts.https .Values.worker.containerPorts.http .Values.security.ssl.enabled }}
to:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }}
app.kubernetes.io/component: worker
# Allow outbound connections to other master pods
- ports:
- port: {{ .Values.service.ports.cluster }}
- port: {{ ternary .Values.service.ports.https .Values.service.ports.http .Values.security.ssl.enabled }}
- port: {{ .Values.master.containerPorts.cluster }}
- port: {{ ternary .Values.master.containerPorts.https .Values.master.containerPorts.http .Values.security.ssl.enabled }}
to:
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }}
app.kubernetes.io/component: master
{{- if .Values.master.networkPolicy.extraEgress }}
{{- include "common.tplvalues.render" ( dict "value" .Values.master.networkPolicy.extraEgress "context" $ ) | nindent 4 }}
{{- end }}
{{- end }}
ingress:
- ports:
- port: {{ .Values.worker.containerPorts.cluster }}
- port: {{ ternary .Values.worker.containerPorts.https .Values.worker.containerPorts.http .Values.security.ssl.enabled }}
{{- if not .Values.worker.networkPolicy.allowExternal }}
from:
- podSelector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 14 }}
- podSelector:
matchLabels:
{{ template "common.names.fullname" . }}-worker: "true"
{{- if .Values.worker.networkPolicy.ingressNSMatchLabels }}
- namespaceSelector:
matchLabels:
{{- range $key, $value := .Values.worker.networkPolicy.ingressNSMatchLabels }}
{{ $key | quote }}: {{ $value | quote }}
{{- end }}
{{- if .Values.worker.networkPolicy.ingressNSPodMatchLabels }}
podSelector:
matchLabels:
{{- range $key, $value := .Values.worker.networkPolicy.ingressNSPodMatchLabels }}
{{ $key | quote }}: {{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.worker.networkPolicy.extraIngress }}
{{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraIngress "context" $ ) | nindent 4 }}
{{- end }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/pdb-master.yaml
Executable
+28
View File
@@ -0,0 +1,28 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.master.pdb.create }}
apiVersion: {{ include "common.capabilities.policy.apiVersion" . }}
kind: PodDisruptionBudget
metadata:
name: {{ printf "%s-master" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: master
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
{{- if .Values.master.pdb.minAvailable }}
minAvailable: {{ .Values.master.pdb.minAvailable }}
{{- end }}
{{- if or .Values.master.pdb.maxUnavailable ( not .Values.master.pdb.minAvailable ) }}
maxUnavailable: {{ .Values.master.pdb.maxUnavailable | default 1 }}
{{- end }}
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: master
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/pdb-worker.yaml
Executable
+28
View File
@@ -0,0 +1,28 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.worker.pdb.create }}
apiVersion: {{ include "common.capabilities.policy.apiVersion" . }}
kind: PodDisruptionBudget
metadata:
name: {{ printf "%s-worker" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: worker
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
{{- if .Values.worker.pdb.minAvailable }}
minAvailable: {{ .Values.worker.pdb.minAvailable }}
{{- end }}
{{- if or .Values.worker.pdb.maxUnavailable ( not .Values.worker.pdb.minAvailable ) }}
maxUnavailable: {{ .Values.worker.pdb.maxUnavailable | default 1 }}
{{- end }}
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: worker
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/podmonitor.yaml
Executable
+49
View File
@@ -0,0 +1,49 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if and .Values.metrics.enabled .Values.metrics.podMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: {{ include "common.names.fullname" . }}
{{- if .Values.metrics.podMonitor.namespace }}
namespace: {{ .Values.metrics.podMonitor.namespace }}
{{- else }}
namespace: {{ include "common.names.namespace" . | quote }}
{{- end }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: metrics
{{- if .Values.metrics.podMonitor.additionalLabels }}
{{- include "common.tplvalues.render" (dict "value" .Values.metrics.podMonitor.additionalLabels "context" $) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
podMetricsEndpoints:
- port: http
path: /metrics/
{{- if .Values.metrics.podMonitor.interval }}
interval: {{ .Values.metrics.podMonitor.interval }}
{{- end }}
{{- if .Values.metrics.podMonitor.scrapeTimeout }}
scrapeTimeout: {{ .Values.metrics.podMonitor.scrapeTimeout }}
{{- end }}
{{- range .Values.metrics.podMonitor.extraMetricsEndpoints }}
- port: {{ .port }}
path: {{ .path }}
{{- if $.Values.metrics.podMonitor.interval }}
interval: {{ $.Values.metrics.podMonitor.interval }}
{{- end }}
{{- if $.Values.metrics.podMonitor.scrapeTimeout }}
scrapeTimeout: {{ $.Values.metrics.podMonitor.scrapeTimeout }}
{{- end }}
{{- end }}
namespaceSelector:
matchNames:
- {{ include "common.names.namespace" . | quote }}
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 6 }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/prometheusrule.yaml
Executable
+24
View File
@@ -0,0 +1,24 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: {{ include "common.names.fullname" . }}
namespace: {{ default (include "common.names.namespace" .) .Values.metrics.prometheusRule.namespace | quote}}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: metrics
{{- if .Values.metrics.prometheusRule.additionalLabels }}
{{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $ ) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
groups:
- name: {{ include "common.names.fullname" . }}
rules: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.rules "context" $ ) | nindent 6 }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/secret.yaml
Executable
+25
View File
@@ -0,0 +1,25 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if not .Values.security.passwordsSecretName }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "spark.passwordsSecretName" . }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- if .Values.security.rpc.authenticationEnabled }}
rpc-authentication-secret: {{ "RPCDefaultSecret" | b64enc | quote }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
ssl-keystore-password: {{ (.Values.security.ssl.keystorePassword | default (randAlphaNum 10)) | b64enc | quote }}
ssl-truststore-password: {{ (.Values.security.ssl.truststorePassword | default (randAlphaNum 10)) | b64enc | quote }}
{{- end }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/serviceaccount.yaml
Executable
+18
View File
@@ -0,0 +1,18 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "spark.serviceAccountName" . }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/statefulset-master.yaml
Executable
+373
View File
@@ -0,0 +1,373 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
kind: StatefulSet
metadata:
name: {{ printf "%s-master" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: master
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
serviceName: {{ printf "%s-headless" (include "common.names.fullname" .) }}
replicas: 1
{{- if .Values.master.updateStrategy }}
updateStrategy: {{- toYaml .Values.master.updateStrategy | nindent 4 }}
{{- end }}
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: master
template:
metadata:
labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
app.kubernetes.io/component: master
{{- if or .Values.master.podAnnotations (and .Values.metrics.enabled .Values.metrics.masterAnnotations) }}
annotations:
{{- if .Values.master.podAnnotations }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.podAnnotations "context" $) | nindent 8 }}
{{- end }}
{{- if and .Values.metrics.enabled .Values.metrics.masterAnnotations }}
{{- include "common.tplvalues.render" ( dict "value" .Values.metrics.masterAnnotations "context" $) | nindent 8 }}
{{- end }}
{{- end }}
spec:
{{- include "spark.imagePullSecrets" . | nindent 6 }}
serviceAccountName: {{ template "spark.serviceAccountName" . }}
{{- if .Values.master.affinity }}
affinity: {{- include "common.tplvalues.render" (dict "value" .Values.master.affinity "context" $) | nindent 8 }}
{{- else }}
affinity:
podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.master.podAffinityPreset "component" "master" "customLabels" $podLabels "context" $) | nindent 10 }}
podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.master.podAntiAffinityPreset "component" "master" "customLabels" $podLabels "context" $) | nindent 10 }}
nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.master.nodeAffinityPreset.type "key" .Values.master.nodeAffinityPreset.key "values" .Values.master.nodeAffinityPreset.values) | nindent 10 }}
{{- end }}
automountServiceAccountToken: {{ .Values.master.automountServiceAccountToken }}
{{- if .Values.master.hostAliases }}
hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.master.hostAliases "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.master.nodeSelector }}
nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.master.nodeSelector "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.master.tolerations }}
tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.master.tolerations "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.master.priorityClassName }}
priorityClassName: {{ .Values.master.priorityClassName | quote }}
{{- end }}
{{- if .Values.master.schedulerName }}
schedulerName: {{ .Values.master.schedulerName | quote }}
{{- end }}
{{- if .Values.master.topologySpreadConstraints }}
topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.master.topologySpreadConstraints "context" .) | nindent 8 }}
{{- end }}
{{- if .Values.master.podSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.master.podSecurityContext "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.master.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.master.terminationGracePeriodSeconds }}
{{- end }}
{{- if .Values.hostNetwork }}
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
{{- end }}
{{- if or .Values.master.initContainers .Values.security.ssl.enabled }}
initContainers:
{{- if .Values.master.initContainers }}
{{ include "common.tplvalues.render" (dict "value" .Values.master.initContainers "context" $) | trim | nindent 8 }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: init-certs
image: {{ include "spark.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- /bin/bash
- -ec
- |-
{{- if .Values.security.ssl.autoGenerated }}
if [[ -f "/certs/tls.key" ]] && [[ -f "/certs/tls.crt" ]] && [[ -f "/certs/ca.crt" ]]; then
openssl pkcs12 -export -in "/certs/tls.crt" \
-passout pass:"${SPARK_SSL_KEYSTORE_PASSWORD}" \
-inkey "/certs/tls.key" \
-out "/tmp/keystore.p12"
keytool -importkeystore -srckeystore "/tmp/keystore.p12" \
-srcstoretype PKCS12 \
-srcstorepass "${SPARK_SSL_KEYSTORE_PASSWORD}" \
-deststorepass "${SPARK_SSL_KEYSTORE_PASSWORD}" \
-destkeystore "/opt/bitnami/spark/conf/certs/spark-keystore.jks"
rm "/tmp/keystore.p12"
keytool -import -file "/certs/ca.crt" \
-keystore "/opt/bitnami/spark/conf/certs/spark-truststore.jks" \
-storepass "${SPARK_SSL_TRUSTSTORE_PASSWORD}" \
-noprompt
else
echo "Couldn't find the expected PEM certificates! They are mandatory when encryption via TLS is enabled."
exit 1
fi
{{- else }}
if [[ -f "/certs/spark-truststore.jks" ]] && [[ -f "/certs/spark-keystore.jks" ]]; then
cp "/certs/spark-truststore.jks" "/opt/bitnami/spark/conf/certs/spark-truststore.jks"
cp "/certs/spark-keystore.jks" "/opt/bitnami/spark/conf/certs/spark-keystore.jks"
else
echo "Couldn't find the expected Java Key Stores (JKS) files! They are mandatory when encryption via TLS is enabled."
exit 1
fi
{{- end }}
env:
- name: SPARK_SSL_KEY_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_KEYSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_TRUSTSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-truststore-password
{{- if .Values.security.ssl.resources }}
resources: {{- toYaml .Values.security.ssl.resources | nindent 12 }}
{{- else if ne .Values.security.ssl.resourcesPreset "none" }}
resources: {{- include "common.resources.preset" (dict "type" .Values.security.ssl.resourcesPreset) | nindent 12 }}
{{- end }}
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: certs
mountPath: /certs
- name: shared-certs
mountPath: /opt/bitnami/spark/conf/certs
{{- end }}
{{- end }}
containers:
- name: spark-master
image: {{ include "spark.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
{{- if .Values.master.containerSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.master.containerSecurityContext "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.diagnosticMode.enabled }}
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
{{- else if .Values.master.command }}
command: {{- include "common.tplvalues.render" (dict "value" .Values.master.command "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.diagnosticMode.enabled }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
{{- else if .Values.master.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.master.args "context" $) | nindent 12 }}
{{- end }}
ports:
{{- if .Values.security.ssl.enabled }}
- name: https
containerPort: {{ .Values.master.containerPorts.https }}
protocol: TCP
{{- else }}
- name: http
containerPort: {{ .Values.master.containerPorts.http }}
protocol: TCP
{{- end }}
- name: cluster
containerPort: {{ .Values.master.containerPorts.cluster }}
{{- if .Values.master.extraContainerPorts }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.extraContainerPorts "context" $) | nindent 12 }}
{{- end }}
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/conf
subPath: app-conf-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/tmp
subPath: app-tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/logs
subPath: app-logs-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/work
subPath: app-work-dir
{{- if .Values.master.existingConfigmap }}
- name: config
mountPath: /bitnami/spark/conf/
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: shared-certs
mountPath: /opt/bitnami/spark/conf/certs
readOnly: true
{{- end }}
{{- if .Values.master.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.extraVolumeMounts "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.initScripts }}
- name: custom-init-scripts
mountPath: /docker-entrypoint-initdb.d/init-scripts
{{- end }}
{{- if .Values.initScriptsCM }}
- name: custom-init-scripts-cm
mountPath: /docker-entrypoint-initdb.d/init-scripts-cm
{{- end }}
{{- if .Values.initScriptsSecret }}
- name: custom-init-scripts-secret
mountPath: /docker-entrypoint-initdb.d/init-scripts-secret
{{- end }}
env:
- name: BITNAMI_DEBUG
value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
- name: SPARK_MODE
value: "master"
{{- if .Values.metrics.enabled }}
- name: SPARK_METRICS_ENABLED
value: "true"
{{- end }}
- name: SPARK_DAEMON_MEMORY
value: {{ .Values.master.daemonMemoryLimit | quote }}
{{- if .Values.master.containerPorts.cluster }}
- name: SPARK_MASTER_PORT
value: {{ .Values.master.containerPorts.cluster | quote }}
{{- end }}
{{- if .Values.master.containerPorts.http }}
- name: SPARK_MASTER_WEBUI_PORT
value: {{ .Values.master.containerPorts.http | quote }}
{{- end }}
{{- if .Values.master.configOptions }}
- name: SPARK_MASTER_OPTS
value: {{ .Values.master.configOptions | quote }}
{{- end }}
{{- if .Values.security.rpc.authenticationEnabled }}
- name: SPARK_RPC_AUTHENTICATION_ENABLED
value: {{ .Values.security.rpc.authenticationEnabled | quote }}
- name: SPARK_RPC_AUTHENTICATION_SECRET
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: rpc-authentication-secret
{{- end }}
{{- if .Values.security.rpc.encryptionEnabled }}
- name: SPARK_RPC_ENCRYPTION_ENABLED
value: {{ .Values.security.rpc.encryptionEnabled | quote }}
{{- end }}
{{- if .Values.security.storageEncryptionEnabled }}
- name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED
value: {{ .Values.security.storageEncryptionEnabled | quote }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: SPARK_SSL_ENABLED
value: {{ .Values.security.ssl.enabled | quote }}
- name: SPARK_SSL_KEY_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_KEYSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_TRUSTSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-truststore-password
- name: SPARK_SSL_NEED_CLIENT_AUTH
value: {{ .Values.security.ssl.needClientAuth | quote }}
- name: SPARK_SSL_PROTOCOL
value: {{ .Values.security.ssl.protocol | quote }}
- name: SPARK_WEBUI_SSL_PORT
value: {{ .Values.master.containerPorts.https | quote }}
{{- end }}
{{- if .Values.master.extraEnvVars }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.extraEnvVars "context" $) | nindent 12 }}
{{- end }}
envFrom:
{{- if .Values.master.extraEnvVarsCM }}
- configMapRef:
name: {{ include "common.tplvalues.render" (dict "value" .Values.master.extraEnvVarsCM "context" $) }}
{{- end }}
{{- if .Values.master.extraEnvVarsSecret }}
- secretRef:
name: {{ include "common.tplvalues.render" (dict "value" .Values.master.extraEnvVarsSecret "context" $) }}
{{- end }}
{{- if not .Values.diagnosticMode.enabled }}
{{- if .Values.master.customLivenessProbe }}
livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.master.customLivenessProbe "context" $) | nindent 12 }}
{{- else if .Values.master.livenessProbe.enabled }}
livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.master.livenessProbe "enabled") "context" $) | nindent 12 }}
tcpSocket:
port: {{ .Values.master.containerPorts.http }}
{{- end }}
{{- if .Values.master.customReadinessProbe }}
readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.master.customReadinessProbe "context" $) | nindent 12 }}
{{- else if .Values.master.readinessProbe.enabled }}
readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.master.readinessProbe "enabled") "context" $) | nindent 12 }}
httpGet:
path: /
port: {{ .Values.master.containerPorts.http }}
{{- end }}
{{- if .Values.master.customStartupProbe }}
startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.master.customStartupProbe "context" $) | nindent 12 }}
{{- else if .Values.master.startupProbe.enabled }}
startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.master.startupProbe "enabled") "context" $) | nindent 12 }}
httpGet:
path: /
port: {{ .Values.master.containerPorts.http }}
{{- end }}
{{- end }}
{{- if .Values.master.lifecycleHooks }}
lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.master.lifecycleHooks "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.master.resources }}
resources: {{- toYaml .Values.master.resources | nindent 12 }}
{{- else if ne .Values.master.resourcesPreset "none" }}
resources: {{- include "common.resources.preset" (dict "type" .Values.master.resourcesPreset) | nindent 12 }}
{{- end }}
{{- if .Values.master.sidecars }}
{{- include "common.tplvalues.render" ( dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
- name: empty-dir
emptyDir: {}
{{- if .Values.master.existingConfigmap }}
- name: config
configMap:
name: {{ .Values.master.existingConfigmap }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: certs
secret:
secretName: {{ include "spark.tlsSecretName" . }}
- name: shared-certs
emptyDir: {}
{{- end }}
{{- if .Values.master.extraVolumes }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.extraVolumes "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.initScripts }}
- name: custom-init-scripts
configMap:
name: {{ template "spark.initScripts" . }}
{{- end }}
{{- if .Values.initScriptsCM }}
- name: custom-init-scripts-cm
configMap:
name: {{ template ".Values.initScriptsCM" . }}
{{- end }}
{{- if .Values.initScriptsSecret }}
- name: custom-init-scripts-secret
secret:
secretName: {{ template ".Values.initScriptsSecret" . }}
defaultMode: 0755
{{- end }}
{{- if .Values.master.extraVolumeClaimTemplates }}
volumeClaimTemplates: {{- include "common.tplvalues.render" (dict "value" .Values.master.extraVolumeClaimTemplates "context" $) | nindent 8 }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/statefulset-worker.yaml
Executable
+397
View File
@@ -0,0 +1,397 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
kind: StatefulSet
metadata:
name: {{ printf "%s-worker" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: worker
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
serviceName: {{ printf "%s-headless" (include "common.names.fullname" .) }}
replicas: {{ .Values.worker.replicaCount }}
podManagementPolicy: {{ .Values.worker.podManagementPolicy }}
{{- if .Values.worker.updateStrategy }}
updateStrategy: {{- toYaml .Values.worker.updateStrategy | nindent 4 }}
{{- end }}
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.podLabels .Values.commonLabels ) "context" . ) }}
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: worker
template:
metadata:
labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }}
app.kubernetes.io/component: worker
{{- if or .Values.worker.podAnnotations (and .Values.metrics.enabled .Values.metrics.workerAnnotations) }}
annotations:
{{- if .Values.worker.podAnnotations }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }}
{{- end }}
{{- if and .Values.metrics.enabled .Values.metrics.workerAnnotations }}
{{- include "common.tplvalues.render" ( dict "value" .Values.metrics.workerAnnotations "context" $) | nindent 8 }}
{{- end }}
{{- end }}
spec:
{{- include "spark.imagePullSecrets" . | nindent 6 }}
serviceAccountName: {{ template "spark.serviceAccountName" . }}
{{- if .Values.worker.affinity }}
affinity: {{- include "common.tplvalues.render" (dict "value" .Values.worker.affinity "context" $) | nindent 8 }}
{{- else }}
affinity:
podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAffinityPreset "component" "worker" "customLabels" $podLabels "context" $) | nindent 10 }}
podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.worker.podAntiAffinityPreset "component" "worker" "customLabels" $podLabels "context" $) | nindent 10 }}
nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.worker.nodeAffinityPreset.type "key" .Values.worker.nodeAffinityPreset.key "values" .Values.worker.nodeAffinityPreset.values) | nindent 10 }}
{{- end }}
automountServiceAccountToken: {{ .Values.worker.automountServiceAccountToken }}
{{- if .Values.worker.hostAliases }}
hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.worker.hostAliases "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.worker.nodeSelector }}
nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.worker.tolerations }}
tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.worker.tolerations "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.worker.priorityClassName }}
priorityClassName: {{ .Values.worker.priorityClassName | quote }}
{{- end }}
{{- if .Values.worker.schedulerName }}
schedulerName: {{ .Values.worker.schedulerName | quote }}
{{- end }}
{{- if .Values.worker.topologySpreadConstraints }}
topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.worker.topologySpreadConstraints "context" .) | nindent 8 }}
{{- end }}
{{- if .Values.worker.podSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.podSecurityContext "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.worker.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
{{- end }}
{{- if .Values.hostNetwork }}
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
{{- end }}
{{- if or .Values.worker.initContainers .Values.security.ssl.enabled }}
initContainers:
{{- if .Values.worker.initContainers }}
{{ include "common.tplvalues.render" (dict "value" .Values.worker.initContainers "context" $) | trim | nindent 8 }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: init-certs
image: {{ include "spark.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
command:
- /bin/bash
- -ec
- |-
{{- if .Values.security.ssl.autoGenerated }}
if [[ -f "/certs/tls.key" ]] && [[ -f "/certs/tls.crt" ]] && [[ -f "/certs/ca.crt" ]]; then
openssl rsa -aes256 -in /certs/tls.key -out /tmp/tls.key --passout pass:"$SPARK_SSL_KEY_PASSWORD"
openssl pkcs12 -export -in "/certs/tls.crt" \
-passout pass:"$SPARK_SSL_KEYSTORE_PASSWORD" \
-inkey "/certs/tls.key" \
-out "/tmp/keystore.p12"
keytool -importkeystore -srckeystore "/tmp/keystore.p12" \
-srcstoretype PKCS12 \
-srcstorepass "$SPARK_SSL_KEYSTORE_PASSWORD" \
-deststorepass "$SPARK_SSL_KEYSTORE_PASSWORD" \
-destkeystore "/opt/bitnami/spark/conf/certs/spark-keystore.jks"
rm "/tmp/keystore.p12"
keytool -import -file "/certs/ca.crt" \
-keystore "/opt/bitnami/spark/conf/certs/spark-truststore.jks" \
-storepass "$SPARK_SSL_TRUSTSTORE_PASSWORD" \
-noprompt
else
echo "Couldn't find the expected PEM certificates! They are mandatory when encryption via TLS is enabled."
exit 1
fi
{{- else }}
if [[ -f "/certs/spark-truststore.jks" ]] && [[ -f "/certs/spark-keystore.jks" ]]; then
cp "/certs/spark-truststore.jks" "/opt/bitnami/spark/conf/certs/spark-truststore.jks"
cp "/certs/spark-keystore.jks" "/opt/bitnami/spark/conf/certs/spark-keystore.jks"
else
echo "Couldn't find the expected Java Key Stores (JKS) files! They are mandatory when encryption via TLS is enabled."
exit 1
fi
{{- end }}
env:
- name: SPARK_SSL_KEY_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_KEYSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_TRUSTSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-truststore-password
{{- if .Values.security.ssl.resources }}
resources: {{- toYaml .Values.security.ssl.resources | nindent 12 }}
{{- else if ne .Values.security.ssl.resourcesPreset "none" }}
resources: {{- include "common.resources.preset" (dict "type" .Values.security.ssl.resourcesPreset) | nindent 12 }}
{{- end }}
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: certs
mountPath: /certs
- name: shared-certs
mountPath: /opt/bitnami/spark/conf/certs
{{- end }}
{{- end }}
containers:
- name: spark-worker
image: {{ include "spark.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
{{- if .Values.worker.containerSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.worker.containerSecurityContext "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.diagnosticMode.enabled }}
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
{{- else if .Values.worker.command }}
command: {{- include "common.tplvalues.render" (dict "value" .Values.worker.command "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.diagnosticMode.enabled }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
{{- else if .Values.worker.args }}
args: {{- include "common.tplvalues.render" (dict "value" .Values.worker.args "context" $) | nindent 12 }}
{{- end }}
ports:
{{- if .Values.security.ssl.enabled }}
- name: https
containerPort: {{ .Values.worker.containerPorts.https }}
protocol: TCP
{{- else }}
- name: http
containerPort: {{ .Values.worker.containerPorts.http }}
protocol: TCP
{{- end }}
{{- if .Values.worker.containerPorts.cluster }}
- name: cluster
containerPort: {{ .Values.worker.containerPorts.cluster }}
{{- end }}
{{- if .Values.worker.extraContainerPorts }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.extraContainerPorts "context" $) | nindent 12 }}
{{- end }}
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/conf
subPath: app-conf-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/tmp
subPath: app-tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/logs
subPath: app-logs-dir
- name: empty-dir
mountPath: /opt/bitnami/spark/work
subPath: app-work-dir
{{- if .Values.worker.existingConfigmap }}
- name: config
mountPath: '/bitnami/spark/conf/'
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: shared-certs
mountPath: '/opt/bitnami/spark/conf/certs'
readOnly: true
- name: certs
mountPath: /certs
{{- end }}
{{- if .Values.worker.extraVolumeMounts }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.initScripts }}
- name: custom-init-scripts
mountPath: /docker-entrypoint-initdb.d/init-scripts
{{- end }}
{{- if .Values.initScriptsCM }}
- name: custom-init-scripts-cm
mountPath: /docker-entrypoint-initdb.d/init-scripts-cm
{{- end }}
{{- if .Values.initScriptsSecret }}
- name: custom-init-scripts-secret
mountPath: /docker-entrypoint-initdb.d/init-scripts-secret
{{- end }}
env:
- name: SPARK_MODE
value: "worker"
- name: BITNAMI_DEBUG
value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
- name: SPARK_DAEMON_MEMORY
value: {{ .Values.worker.daemonMemoryLimit | quote }}
{{- if .Values.metrics.enabled }}
- name: SPARK_METRICS_ENABLED
value: "true"
{{- end }}
## There are some environment variables whose existence needs
## to be checked because Spark checks if they are null instead of an
## empty string
{{- if .Values.worker.memoryLimit }}
- name: SPARK_WORKER_MEMORY
value: {{ .Values.worker.memoryLimit | quote }}
{{- end }}
{{- if .Values.worker.coreLimit }}
- name: SPARK_WORKER_CORES
value: {{ .Values.worker.coreLimit | quote }}
{{- end }}
{{- if .Values.worker.containerPorts.cluster }}
- name: SPARK_WORKER_PORT
value: {{ .Values.worker.containerPorts.cluster | quote }}
{{- end }}
{{- if .Values.worker.containerPorts.http }}
- name: SPARK_WORKER_WEBUI_PORT
value: {{ .Values.worker.containerPorts.http | quote }}
{{- end }}
{{- if .Values.worker.dir }}
- name: SPARK_WORKER_DIR
value: {{ .Values.worker.dir | quote }}
{{- end }}
- name: SPARK_DAEMON_JAVA_OPTS
value: {{ .Values.worker.javaOptions | quote }}
- name: SPARK_MASTER_URL
value: {{ printf "spark://%s-master-svc:%d" (include "common.names.fullname" .) (int .Values.service.ports.cluster) }}
# If you use a custom properties file, it must be loaded using a ConfigMap
- name: SPARK_WORKER_OPTS
value: {{ .Values.worker.configOptions }}
{{- if .Values.security.rpc.authenticationEnabled }}
- name: SPARK_RPC_AUTHENTICATION_ENABLED
value: {{ .Values.security.rpc.authenticationEnabled | quote }}
- name: SPARK_RPC_AUTHENTICATION_SECRET
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: rpc-authentication-secret
{{- end }}
{{- if .Values.security.rpc.encryptionEnabled }}
- name: SPARK_RPC_ENCRYPTION_ENABLED
value: {{ .Values.security.rpc.encryptionEnabled | quote }}
{{- end }}
{{- if .Values.security.storageEncryptionEnabled }}
- name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED
value: {{ .Values.security.storageEncryptionEnabled | quote }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: SPARK_SSL_ENABLED
value: {{ .Values.security.ssl.enabled | quote }}
- name: SPARK_SSL_KEY_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_KEYSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-keystore-password
- name: SPARK_SSL_TRUSTSTORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "spark.passwordsSecretName" . }}
key: ssl-truststore-password
- name: SPARK_SSL_NEED_CLIENT_AUTH
value: {{ .Values.security.ssl.needClientAuth | quote }}
- name: SPARK_SSL_PROTOCOL
value: {{ .Values.security.ssl.protocol | quote }}
- name: SPARK_WEBUI_SSL_PORT
value: {{ .Values.worker.containerPorts.https | quote }}
{{- end }}
{{- if .Values.worker.extraEnvVars }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVars "context" $) | nindent 12 }}
{{- end }}
envFrom:
{{- if .Values.worker.extraEnvVarsCM }}
- configMapRef:
name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsCM "context" $) }}
{{- end }}
{{- if .Values.worker.extraEnvVarsSecret }}
- secretRef:
name: {{ include "common.tplvalues.render" (dict "value" .Values.worker.extraEnvVarsSecret "context" $) }}
{{- end }}
{{- if not .Values.diagnosticMode.enabled }}
{{- if .Values.worker.customLivenessProbe }}
livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }}
{{- else if .Values.worker.livenessProbe.enabled }}
livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }}
tcpSocket:
port: {{ .Values.worker.containerPorts.http }}
{{- end }}
{{- if .Values.worker.customReadinessProbe }}
readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }}
{{- else if .Values.worker.readinessProbe.enabled }}
readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }}
httpGet:
path: /
port: {{ .Values.worker.containerPorts.http }}
{{- end }}
{{- if .Values.worker.customStartupProbe }}
startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customStartupProbe "context" $) | nindent 12 }}
{{- else if .Values.worker.startupProbe.enabled }}
startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.startupProbe "enabled") "context" $) | nindent 12 }}
httpGet:
path: /
port: {{ .Values.worker.containerPorts.http }}
{{- end }}
{{- end }}
{{- if .Values.worker.lifecycleHooks }}
lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }}
{{- end }}
{{- if .Values.worker.resources }}
resources: {{- toYaml .Values.worker.resources | nindent 12 }}
{{- else if ne .Values.worker.resourcesPreset "none" }}
resources: {{- include "common.resources.preset" (dict "type" .Values.worker.resourcesPreset) | nindent 12 }}
{{- end }}
{{- if .Values.worker.sidecars }}
{{- include "common.tplvalues.render" ( dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
- name: empty-dir
emptyDir: {}
{{- if .Values.worker.existingConfigmap }}
- name: config
configMap:
name: {{ .Values.worker.existingConfigmap }}
{{- end }}
{{- if .Values.security.ssl.enabled }}
- name: certs
secret:
secretName: {{ include "spark.tlsSecretName" . }}
- name: shared-certs
emptyDir: {}
{{- end }}
{{- if .Values.worker.extraVolumes }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.initScripts }}
- name: custom-init-scripts
configMap:
name: {{ template "spark.initScripts" . }}
{{- end }}
{{- if .Values.initScriptsCM }}
- name: custom-init-scripts-cm
configMap:
name: {{ template ".Values.initScriptsCM" . }}
{{- end }}
{{- if .Values.initScriptsSecret }}
- name: custom-init-scripts-secret
secret:
secretName: {{ template ".Values.initScriptsSecret" . }}
defaultMode: 0755
{{- end }}
{{- if .Values.worker.extraVolumeClaimTemplates }}
volumeClaimTemplates: {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeClaimTemplates "context" $) | nindent 8 }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/templates/svc-master.yaml
Executable
+72
View File
@@ -0,0 +1,72 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
apiVersion: v1
kind: Service
metadata:
name: {{ printf "%s-master-svc" (include "common.names.fullname" .) }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: master
{{- if or .Values.commonAnnotations .Values.service.annotations }}
{{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.service.annotations .Values.commonAnnotations ) "context" . ) }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
{{- if and .Values.service.clusterIP (eq .Values.service.type "ClusterIP") }}
clusterIP: {{ .Values.service.clusterIP }}
{{- end }}
{{- if .Values.service.sessionAffinity }}
sessionAffinity: {{ .Values.service.sessionAffinity }}
{{- end }}
{{- if .Values.service.sessionAffinityConfig }}
sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.service.sessionAffinityConfig "context" $) | nindent 4 }}
{{- end }}
{{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }}
externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }}
{{- end }}
{{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }}
loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }}
{{- end }}
{{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }}
loadBalancerIP: {{ .Values.service.loadBalancerIP }}
{{- end }}
ports:
- port: {{ .Values.service.ports.cluster }}
targetPort: cluster
name: cluster
{{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.cluster)) }}
nodePort: {{ .Values.service.nodePorts.cluster }}
{{- else if eq .Values.service.type "ClusterIP" }}
nodePort: null
{{- end }}
{{- if .Values.security.ssl.enabled }}
- port: {{ .Values.service.ports.https }}
targetPort: https
name: https
protocol: TCP
{{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.https)) }}
nodePort: {{ .Values.service.nodePorts.https }}
{{- else if eq .Values.service.type "ClusterIP" }}
nodePort: null
{{- end }}
{{- else }}
- port: {{ .Values.service.ports.http }}
targetPort: http
name: http
protocol: TCP
{{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.http)) }}
nodePort: {{ .Values.service.nodePorts.http }}
{{- else if eq .Values.service.type "ClusterIP" }}
nodePort: null
{{- end }}
{{- end }}
{{- if .Values.service.extraPorts }}
{{- include "common.tplvalues.render" (dict "value" .Values.service.extraPorts "context" $) | nindent 4 }}
{{- end }}
{{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.podLabels .Values.commonLabels ) "context" . ) }}
selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/component: master
spark-cluster/base/charts/spark-9.2.10/spark/templates/tls-secrets.yaml
Executable
+69
View File
@@ -0,0 +1,69 @@
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if .Values.ingress.enabled }}
{{- if .Values.ingress.secrets }}
{{- range .Values.ingress.secrets }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .name }}
namespace: {{ include "common.names.namespace" $ | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }}
{{- if $.Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: kubernetes.io/tls
data:
tls.crt: {{ .certificate | b64enc }}
tls.key: {{ .key | b64enc }}
---
{{- end }}
{{- end }}
{{- if and .Values.ingress.tls .Values.ingress.selfSigned }}
{{- $secretName := printf "%s-tls" .Values.ingress.hostname }}
{{- $ca := genCA "spark-ca" 365 }}
{{- $cert := genSignedCert .Values.ingress.hostname nil (list .Values.ingress.hostname) 365 $ca }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: kubernetes.io/tls
data:
tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }}
tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }}
ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }}
{{- end }}
{{- end }}
{{- if (include "spark.createTlsSecret" . ) }}
{{- $secretName := printf "%s-crt" (include "common.names.fullname" .) }}
{{- $ca := genCA "spark-internal-ca" 365 }}
{{- $releaseNamespace := include "common.names.namespace" . }}
{{- $clusterDomain := .Values.clusterDomain }}
{{- $fullname := include "common.names.fullname" . }}
{{- $headlessServiceName := printf "%s-headless" ( include "common.names.fullname" . ) }}
{{- $masterServiceName := printf "%s-master-svc" (include "common.names.fullname" .) }}
{{- $altNames := list (printf "*.%s.%s.svc.%s" $masterServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $masterServiceName $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) $fullname }}
{{- $cert := genSignedCert $fullname nil $altNames 365 $ca }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: kubernetes.io/tls
data:
tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }}
tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }}
ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }}
{{- end }}
spark-cluster/base/charts/spark-9.2.10/spark/values.yaml
Executable
+1193
View File
File diff suppressed because it is too large Load Diff
spark-cluster/base/history-server.yaml
Executable
+61
View File
@@ -0,0 +1,61 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spark-history-server
labels:
app.kubernetes.io/name: spark-history-server
app.kubernetes.io/part-of: nowchess
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: spark-history-server
template:
metadata:
labels:
app.kubernetes.io/name: spark-history-server
app.kubernetes.io/part-of: nowchess
spec:
containers:
- name: history-server
# Same image version the Bitnami chart deploys for master/workers.
image: docker.io/bitnami/spark:3.5.2
command:
- /opt/bitnami/spark/bin/spark-class
- org.apache.spark.deploy.history.HistoryServer
env:
- name: SPARK_HISTORY_OPTS
value: "-Dspark.history.fs.logDirectory=/spark-events"
ports:
- name: http
containerPort: 18080
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumeMounts:
- name: spark-events
mountPath: /spark-events
volumes:
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
---
apiVersion: v1
kind: Service
metadata:
name: spark-history-server-svc
labels:
app.kubernetes.io/name: spark-history-server
app.kubernetes.io/part-of: nowchess
spec:
selector:
app.kubernetes.io/name: spark-history-server
ports:
- name: http
port: 18080
targetPort: http
type: ClusterIP
spark-cluster/base/ingress.yaml
Executable
+46
View File
@@ -0,0 +1,46 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spark-master-ingress
labels:
app.kubernetes.io/name: spark-cluster
app.kubernetes.io/part-of: nowchess
spec:
ingressClassName: traefik
rules:
- host: spark.nowchess.janis-eccarius.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: spark-master-svc
port:
number: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spark-history-ingress
labels:
app.kubernetes.io/name: spark-cluster
app.kubernetes.io/part-of: nowchess
annotations:
# NOTE: The Bitnami spark chart 9.2.10 does not include a standalone history
# server deployment or service. This Ingress is a placeholder. To activate
# history server access, deploy a separate spark-history-server Deployment
# and Service named spark-history-server-svc on port 18080.
spec:
ingressClassName: traefik
rules:
- host: spark-history.nowchess.janis-eccarius.de
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: spark-history-server-svc
port:
number: 18080
spark-cluster/base/kustomization.yaml
Executable
+60
View File
@@ -0,0 +1,60 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- pvc.yaml
- history-server.yaml
- ingress.yaml
helmCharts:
- name: spark
repo: https://charts.bitnami.com/bitnami
version: "9.2.10"
releaseName: spark
namespace: nowchess
valuesInline:
master:
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
extraVolumes:
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
extraVolumeMounts:
- name: spark-events
mountPath: /spark-events
worker:
replicaCount: 2
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
extraVolumes:
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
extraVolumeMounts:
- name: spark-events
mountPath: /spark-events
historyServer:
enabled: true
eventsDir: /spark-events
extraVolumes:
- name: spark-events
persistentVolumeClaim:
claimName: spark-events
extraVolumeMounts:
- name: spark-events
mountPath: /spark-events
ingress:
enabled: false
service:
type: ClusterIP
spark-cluster/base/pvc.yaml
Executable
+18
View File
@@ -0,0 +1,18 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: spark-events
labels:
app.kubernetes.io/name: spark-cluster
app.kubernetes.io/part-of: nowchess
spec:
# ReadWriteOnce for k3d/local-path compatibility.
# NOTE: k3d uses the local-path provisioner which only supports RWO.
# For production with multiple concurrent writers (workers + history server),
# replace with a RWX-capable storage class such as NFS or Longhorn and
# change accessModes to [ReadWriteMany].
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
spark-cluster/eu-central-1-prod/kustomization.yaml
Executable
+5
View File
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: nowchess
resources:
- ../base