Skip to content

Commit 7462b93

Browse files
KunWuLuanKunWuLuan
committed
add samples for minio
Signed-off-by: KunWuLuan <[email protected]>
1 parent b97507f commit 7462b93

File tree

4 files changed

+390
-0
lines changed

4 files changed

+390
-0
lines changed

‎historyserver/config/samples/minio/historyserver.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: historyserver #TODO: to specify your service name
6+
labels:
7+
app: historyserver
8+
spec:
9+
selector:
10+
app: historyserver
11+
ports:
12+
- protocol: TCP
13+
name: http
14+
port: 30080
15+
targetPort: 8080
16+
type: ClusterIP
17+
---
18+
apiVersion: apps/v1 # for versions before 1.8.0 use apps/v1beta1
19+
kind: Deployment
20+
metadata:
21+
name: historyserver-demo
22+
labels:
23+
app: historyserver
24+
spec:
25+
replicas: 1
26+
selector:
27+
matchLabels:
28+
app: historyserver
29+
template:
30+
metadata:
31+
labels:
32+
app: historyserver
33+
spec:
34+
imagePullSecrets:
35+
containers:
36+
- name: historyserver
37+
env:
38+
- name: S3DISABLE_SSL
39+
value: "true"
40+
- name: AWS_S3ID
41+
value: minioadmin
42+
- name: AWS_S3SECRET
43+
value: minioadmin
44+
- name: AWS_S3TOKEN
45+
value: ""
46+
- name: S3_BUCKET
47+
value: "ray-historyserver-log"
48+
- name: S3_ENDPOINT
49+
value: "minio-service.minio-dev:9000"
50+
- name: S3_REGION
51+
value: "test"
52+
- name: S3FORCE_PATH_STYPE
53+
value: "true"
54+
image: xxx
55+
imagePullPolicy: Always
56+
command:
57+
- historyserver
58+
- --runtime-class-name=s3
59+
- --ray-root-dir=log
60+
ports:
61+
- containerPort: 8080
62+
resources:
63+
limits:
64+
cpu: "500m"
65+
volumeMounts:
66+
volumes:

‎historyserver/config/samples/minio/minio.yaml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: minio-dev
5+
---
6+
# 1. Secrets for Login
7+
apiVersion: v1
8+
kind: Secret
9+
metadata:
10+
name: minio-creds
11+
namespace: minio-dev
12+
type: Opaque
13+
stringData:
14+
rootUser: minioadmin # CHANGE THIS
15+
rootPassword: minioadmin # CHANGE THIS
16+
#---
17+
## 2. Storage (PVC)
18+
#apiVersion: v1
19+
#kind: PersistentVolumeClaim
20+
#metadata:
21+
# name: minio-pvc
22+
# namespace: minio-dev
23+
#spec:
24+
# accessModes:
25+
# - ReadWriteOnce
26+
# resources:
27+
# requests:
28+
# storage: 10Gi
29+
---
30+
# 3. Deployment
31+
apiVersion: apps/v1
32+
kind: Deployment
33+
metadata:
34+
name: minio
35+
namespace: minio-dev
36+
labels:
37+
app: minio
38+
spec:
39+
replicas: 1
40+
selector:
41+
matchLabels:
42+
app: minio
43+
template:
44+
metadata:
45+
labels:
46+
app: minio
47+
spec:
48+
containers:
49+
- name: minio
50+
image: quay.io/minio/minio:latest
51+
command:
52+
- /bin/sh
53+
- -c
54+
- minio server /data --console-address :9001
55+
env:
56+
- name: MINIO_ROOT_USER
57+
valueFrom:
58+
secretKeyRef:
59+
name: minio-creds
60+
key: rootUser
61+
- name: MINIO_ROOT_PASSWORD
62+
valueFrom:
63+
secretKeyRef:
64+
name: minio-creds
65+
key: rootPassword
66+
ports:
67+
- containerPort: 9000 # API Port
68+
- containerPort: 9001 # Console UI Port
69+
# volumeMounts:
70+
# - name: data
71+
# mountPath: /data
72+
# volumes:
73+
# - name: data
74+
# persistentVolumeClaim:
75+
# claimName: minio-pvc
76+
---
77+
# 4. Service
78+
apiVersion: v1
79+
kind: Service
80+
metadata:
81+
name: minio-service
82+
namespace: minio-dev
83+
spec:
84+
selector:
85+
app: minio
86+
ports:
87+
- name: api
88+
port: 9000
89+
targetPort: 9000
90+
- name: console
91+
port: 9001
92+
targetPort: 9001
93+
type: ClusterIP # Change to LoadBalancer if on Cloud/MetalLB

‎historyserver/config/samples/minio/raycluster.yaml‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
apiVersion: ray.io/v1
2+
kind: RayCluster
3+
metadata:
4+
labels:
5+
ray.io/cluster: raycluster-historyserver
6+
name: raycluster-historyserver
7+
namespace: default
8+
spec:
9+
headGroupSpec:
10+
rayStartParams:
11+
dashboard-host: 0.0.0.0
12+
num-cpus: "0"
13+
serviceType: ClusterIP
14+
template:
15+
metadata:
16+
labels:
17+
test: raycluster-historyserver
18+
spec:
19+
imagePullSecrets:
20+
affinity:
21+
containers:
22+
- env:
23+
- name: RAY_enable_core_worker_ray_event_to_aggregator
24+
value: "1"
25+
- name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
26+
value: "http://localhost:8084/v1/events"
27+
image: xxx
28+
imagePullPolicy: IfNotPresent
29+
command:
30+
- 'echo "=========================================="; [ -d "/tmp/ray/session_latest" ] && dest="/tmp/ray/prev-logs/$(basename $(readlink /tmp/ray/session_latest))/$(cat /tmp/ray/raylet_node_id)" && echo "dst is $dest" && mkdir -p "$dest" && mv /tmp/ray/session_latest/logs "$dest/logs"; echo "========================================="'
31+
securityContext:
32+
allowPrivilegeEscalation: true # 允许特权提升
33+
privileged: true
34+
name: ray-head
35+
lifecycle:
36+
postStart:
37+
exec:
38+
command:
39+
- /bin/sh
40+
- -lc
41+
- --
42+
- |
43+
GetNodeId(){
44+
while true;
45+
do
46+
nodeid=$(ps -ef | grep raylet | grep node_id | grep -v grep | grep -oP '(?<=--node_id=)[^ ]*')
47+
if [ -n "$nodeid" ]; then
48+
echo "$(date) raylet started: \"$(ps -ef | grep raylet | grep node_id | grep -v grep | grep -oP '(?<=--node_id=)[^ ]*')\" => ${nodeid}" >> /tmp/ray/init.log
49+
echo $nodeid > /tmp/ray/raylet_node_id
50+
break
51+
else
52+
echo "$(date) raylet not start >> /tmp/ray/init.log"
53+
sleep 1
54+
fi
55+
done
56+
}
57+
GetNodeId
58+
resources:
59+
limits:
60+
cpu: "5"
61+
memory: 10G
62+
requests:
63+
cpu: "50m"
64+
memory: 1G
65+
volumeMounts:
66+
- name: historyserver
67+
mountPath: /tmp/ray
68+
- name: collector
69+
image: xxx
70+
imagePullPolicy: Always
71+
env:
72+
- name: S3DISABLE_SSL
73+
value: "true"
74+
- name: AWS_S3ID
75+
value: minioadmin
76+
- name: AWS_S3SECRET
77+
value: minioadmin
78+
- name: AWS_S3TOKEN
79+
value: ""
80+
- name: S3_BUCKET
81+
value: "ray-historyserver-log"
82+
- name: S3_ENDPOINT
83+
value: "minio-service.minio-dev:9000"
84+
- name: S3_REGION
85+
value: "test"
86+
- name: S3FORCE_PATH_STYPE
87+
value: "true"
88+
command:
89+
- collector
90+
- --role=Head
91+
- --runtime-class-name=s3
92+
- --ray-cluster-name=raycluster-historyserver
93+
- --ray-root-dir=log
94+
- --events-port=8084
95+
volumeMounts:
96+
- name: historyserver
97+
mountPath: /tmp/ray
98+
tolerations:
99+
- key: ray
100+
operator: Equal
101+
value: cpu
102+
volumes:
103+
- name: historyserver
104+
emptyDir: {}
105+
workerGroupSpecs:
106+
- groupName: cpu
107+
maxReplicas: 1000
108+
minReplicas: 0
109+
numOfHosts: 1
110+
rayStartParams: {}
111+
replicas: 0
112+
template:
113+
metadata:
114+
labels:
115+
test: raycluster-historyserver
116+
spec:
117+
imagePullSecrets:
118+
containers:
119+
- env:
120+
- name: RAY_enable_core_worker_ray_event_to_aggregator
121+
value: "1"
122+
- name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
123+
value: "http://localhost:8084/v1/events"
124+
image: xxx
125+
command:
126+
- 'echo "=========================================="; [ -d "/tmp/ray/session_latest" ] && dest="/tmp/ray/prev-logs/$(basename $(readlink /tmp/ray/session_latest))/$(cat /tmp/ray/raylet_node_id)" && echo "dst is $dest" && mkdir -p "$dest" && mv /tmp/ray/session_latest/logs "$dest/logs"; echo "========================================="'
127+
imagePullPolicy: IfNotPresent
128+
name: ray-worker
129+
securityContext:
130+
allowPrivilegeEscalation: true # 允许特权提升
131+
privileged: true
132+
lifecycle:
133+
postStart:
134+
exec:
135+
command:
136+
- /bin/sh
137+
- -lc
138+
- --
139+
- |
140+
GetNodeId(){
141+
while true;
142+
do
143+
nodeid=$(ps -ef | grep raylet | grep node_id | grep -v grep | grep -oP '(?<=--node_id=)[^ ]*')
144+
if [ -n "$nodeid" ]; then
145+
echo "$(date) raylet started: \"$(ps -ef | grep raylet | grep node_id | grep -v grep | grep -oP '(?<=--node_id=)[^ ]*')\" => ${nodeid}" >> /tmp/ray/init.log
146+
echo $nodeid > /tmp/ray/raylet_node_id
147+
break
148+
else
149+
echo "$(date) raylet not start >> /tmp/ray/init.log"
150+
sleep 1
151+
fi
152+
done
153+
}
154+
GetNodeId
155+
resources:
156+
limits:
157+
cpu: "30"
158+
memory: 30G
159+
requests:
160+
cpu: "50m"
161+
memory: 1G
162+
volumeMounts:
163+
- name: historyserver
164+
mountPath: /tmp/ray
165+
- name: collector
166+
image: xxx
167+
imagePullPolicy: Always
168+
env:
169+
- name: AWS_S3ID
170+
value: minioadmin
171+
- name: AWS_S3SECRET
172+
value: minioadmin
173+
- name: AWS_S3TOKEN
174+
value: ""
175+
- name: S3_BUCKET
176+
value: "ray-historyserver-log"
177+
- name: S3_ENDPOINT
178+
value: "minio-service.minio-dev:9000"
179+
- name: S3_REGION
180+
value: "test"
181+
- name: S3FORCE_PATH_STYPE
182+
value: "true"
183+
- name: S3DISABLE_SSL
184+
value: "true"
185+
command:
186+
- collector
187+
- --role=Worker
188+
- --runtime-class-name=s3
189+
- --ray-cluster-name=raycluster-historyserver
190+
- --ray-root-dir=log
191+
- --events-port=8084
192+
volumeMounts:
193+
- name: historyserver
194+
mountPath: /tmp/ray
195+
tolerations:
196+
- key: ray
197+
operator: Equal
198+
value: cpu
199+
volumes:
200+
- name: historyserver
201+
emptyDir: {}

‎historyserver/config/samples/minio/sa.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: historyserver
5+
automountServiceAccountToken: true
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: ClusterRole
9+
metadata:
10+
name: raycluster-reader
11+
rules:
12+
- apiGroups: ["ray.io"]
13+
resources: ["rayclusters"]
14+
verbs: ["list", "get"]
15+
---
16+
apiVersion: rbac.authorization.k8s.io/v1
17+
kind: ClusterRoleBinding
18+
metadata:
19+
name: historyserver
20+
namespace: default
21+
subjects:
22+
- kind: ServiceAccount
23+
name: default
24+
namespace: default
25+
- kind: ServiceAccount
26+
name: historyserver
27+
namespace: default
28+
roleRef:
29+
kind: ClusterRole
30+
name: raycluster-reader

0 commit comments

Comments
 (0)