Skip to content

Commit 6e64229

Browse files
committed
Add preheat implementation
Signed-off-by: kerthcet <[email protected]>
1 parent 7d00691 commit 6e64229

26 files changed

+723
-23
lines changed

Dockerfile.preheat

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
ARG BASE_IMAGE
2+
ARG BUILDER_IMAGE
3+
4+
# Build the manager binary
5+
FROM ${BUILDER_IMAGE} as builder
6+
ARG TARGETOS
7+
ARG TARGETARCH
8+
9+
WORKDIR /workspace
10+
# Copy the Go Modules manifests
11+
COPY preheat/go.mod go.mod
12+
COPY preheat/go.sum go.sum
13+
# cache deps before building and copying source so that we don't need to re-download as much
14+
# and so that source changes don't invalidate our downloaded layer
15+
RUN go mod download
16+
17+
# Copy the go source
18+
COPY preheat/main.go main.go
19+
20+
# Build
21+
# the GOARCH has not a default value to allow the binary be built according to the host where the command
22+
# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
23+
# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
24+
# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
25+
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager main.go
26+
27+
# Use distroless as minimal base image to package the manager binary
28+
# Refer to https://github.com/GoogleContainerTools/distroless for more details
29+
FROM ${BASE_IMAGE}
30+
WORKDIR /
31+
COPY --from=builder /workspace/manager .
32+
USER 65532:65532
33+
34+
# Expose the http server.
35+
EXPOSE 9090
36+
37+
ENTRYPOINT ["/manager"]

Makefile

+16
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,12 @@ IMAGE_NAME ?= manta
6363
IMAGE_REPO := $(IMAGE_REGISTRY)/$(IMAGE_NAME)
6464
AGENT_IMAGE_NAME ?= manta-agent
6565
AGENT_IMAGE_REPO := $(IMAGE_REGISTRY)/$(AGENT_IMAGE_NAME)
66+
PREHEAT_IMAGE_NAME ?= manta-preheat
67+
PREHEAT_IMAGE_REPO := $(IMAGE_REGISTRY)/$(PREHEAT_IMAGE_NAME)
6668
GIT_TAG ?= $(shell git describe --tags --dirty --always)
6769
IMG ?= $(IMAGE_REPO):$(GIT_TAG)
6870
AGENT_IMG ?= $(AGENT_IMAGE_REPO):$(GIT_TAG)
71+
PREHEAT_IMG ?= $(PREHEAT_IMAGE_REPO):$(GIT_TAG)
6972
BUILDER_IMAGE ?= golang:$(GO_VERSION)
7073
KIND_CLUSTER_NAME ?= kind
7174

@@ -190,6 +193,19 @@ agent-image-load: agent-image-build
190193
agent-image-push: IMAGE_BUILD_EXTRA_OPTS=--push
191194
agent-image-push: agent-image-build
192195

196+
.PHONY: preheat-image-build
197+
preheat-image-build:
198+
$(IMAGE_BUILD_CMD) -t $(PREHEAT_IMG) \
199+
-f Dockerfile.preheat \
200+
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
201+
--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
202+
--build-arg CGO_ENABLED=$(CGO_ENABLED) \
203+
$(IMAGE_BUILD_EXTRA_OPTS) ./
204+
preheat-image-load: IMAGE_BUILD_EXTRA_OPTS=--load
205+
preheat-image-load: preheat-image-build
206+
preheat-image-push: IMAGE_BUILD_EXTRA_OPTS=--push
207+
preheat-image-push: preheat-image-build
208+
193209
KIND = $(shell pwd)/bin/kind
194210
.PHONY: kind
195211
kind:

agent/config/manager/kustomization.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ kind: Kustomization
55
images:
66
- name: controller
77
newName: inftyai/manta-agent
8-
newTag: v0.0.3
8+
newTag: main

api/v1alpha1/torrent_types.go

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
const (
2626
TorrentNameLabelKey = "manta.io/torrent-name"
2727
TorrentProtectionFinalizer = "manta.io/torrent-protect"
28+
ParentPodNameAnnoKey = "manta.io/parent-pod-name"
2829

2930
HUGGINGFACE_MODEL_HUB = "Huggingface"
3031
)

cmd/main.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@ import (
2020
"flag"
2121
"os"
2222

23-
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
24-
// to ensure that exec-entrypoint and run can make use of them.
25-
2623
_ "k8s.io/client-go/plugin/pkg/client/auth"
2724

2825
"k8s.io/apimachinery/pkg/runtime"
@@ -106,7 +103,6 @@ func main() {
106103
// will block until the cert is ready before setting up the controllers.
107104
// Controllers who register after manager starts will start directly.
108105
go setupControllers(mgr, certsReady)
109-
110106
//+kubebuilder:scaffold:builder
111107

112108
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -161,6 +157,13 @@ func setupControllers(mgr ctrl.Manager, certsReady chan struct{}) {
161157
setupLog.Error(err, "unable to create controller", "controller", "Torrent")
162158
os.Exit(1)
163159
}
160+
if err := controller.NewPodReconciler(
161+
mgr.GetClient(),
162+
mgr.GetScheme(),
163+
).SetupWithManager(mgr); err != nil {
164+
setupLog.Error(err, "unable to create controller", "controller", "Pod")
165+
os.Exit(1)
166+
}
164167

165168
if os.Getenv("ENABLE_WEBHOOKS") != "false" {
166169
if err := webhook.SetupTorrentWebhook(mgr); err != nil {
@@ -171,5 +174,9 @@ func setupControllers(mgr ctrl.Manager, certsReady chan struct{}) {
171174
setupLog.Error(err, "unable to create webhook", "webhook", "Replication")
172175
os.Exit(1)
173176
}
177+
if err := webhook.SetupPodWebhook(mgr); err != nil {
178+
setupLog.Error(err, "unable to create webhook", "webhook", "Pod")
179+
os.Exit(1)
180+
}
174181
}
175182
}

config/crd/bases/manta.io_replications.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ spec:
2121
- jsonPath: .status.phase
2222
name: phase
2323
type: string
24+
- jsonPath: .metadata.creationTimestamp
25+
name: Age
26+
type: date
2427
name: v1alpha1
2528
schema:
2629
openAPIV3Schema:

config/manager/kustomization.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ kind: Kustomization
55
images:
66
- name: controller
77
newName: inftyai/manta
8-
newTag: v0.0.3
8+
newTag: main

config/rbac/role.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ rules:
3131
- list
3232
- update
3333
- watch
34+
- apiGroups:
35+
- ""
36+
resources:
37+
- pods
38+
verbs:
39+
- get
40+
- list
41+
- watch
3442
- apiGroups:
3543
- manta.io
3644
resources:

config/webhook/kustomization.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,11 @@ resources:
44

55
configurations:
66
- kustomizeconfig.yaml
7+
8+
patches:
9+
- path: mutating-patch.yaml
10+
target:
11+
group: admissionregistration.k8s.io
12+
version: v1
13+
kind: MutatingWebhookConfiguration
14+
name: mutating-webhook-configuration

config/webhook/manifests.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,25 @@ kind: MutatingWebhookConfiguration
44
metadata:
55
name: mutating-webhook-configuration
66
webhooks:
7+
- admissionReviewVersions:
8+
- v1
9+
clientConfig:
10+
service:
11+
name: webhook-service
12+
namespace: system
13+
path: /mutate--v1-pod
14+
failurePolicy: Fail
15+
name: mpod.kb.io
16+
rules:
17+
- apiGroups:
18+
- ""
19+
apiVersions:
20+
- v1
21+
operations:
22+
- CREATE
23+
resources:
24+
- pods
25+
sideEffects: None
726
- admissionReviewVersions:
827
- v1
928
clientConfig:

config/webhook/mutating-patch.yaml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
- op: replace
2+
path: /webhooks/0
3+
value:
4+
admissionReviewVersions:
5+
- v1
6+
clientConfig:
7+
service:
8+
name: webhook-service
9+
namespace: system
10+
path: /mutate--v1-pod
11+
failurePolicy: Fail
12+
name: mpod.kb.io
13+
objectSelector:
14+
matchExpressions:
15+
- key: manta.io/torrent-name
16+
operator: Exists
17+
rules:
18+
- apiGroups:
19+
- ""
20+
apiVersions:
21+
- v1
22+
operations:
23+
- CREATE
24+
resources:
25+
- pods
26+
sideEffects: None

hack/e2e-test.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ function cleanup {
2424
then
2525
$KIND delete cluster --name $KIND_CLUSTER_NAME
2626
fi
27-
(cd $CWD/config/manager && $KUSTOMIZE edit set image controller=inftyai/llmaz:main)
27+
(cd $CWD/config/manager && $KUSTOMIZE edit set image controller=inftyai/manta:main)
28+
(cd $CWD/agent/config/manager && $KUSTOMIZE edit set image controller=inftyai/manta-agent:main)
2829
}
2930
function startup {
3031
if [ $USE_EXISTING_CLUSTER == 'false' ]
@@ -44,8 +45,8 @@ function deploy {
4445
# agent
4546
cd $CWD/agent/config/manager && $KUSTOMIZE edit set image controller=$AGENT_IMAGE_TAG
4647
$KUSTOMIZE build $CWD/agent/config | $KUBECTL apply --server-side -f -
47-
4848
}
49+
4950
trap cleanup EXIT
5051
startup
5152
kind_load

pkg/controller/pod_controller.go

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"time"
22+
23+
corev1 "k8s.io/api/core/v1"
24+
"k8s.io/apimachinery/pkg/runtime"
25+
"k8s.io/apimachinery/pkg/types"
26+
"k8s.io/klog/v2"
27+
"k8s.io/utils/ptr"
28+
ctrl "sigs.k8s.io/controller-runtime"
29+
"sigs.k8s.io/controller-runtime/pkg/client"
30+
"sigs.k8s.io/controller-runtime/pkg/event"
31+
"sigs.k8s.io/controller-runtime/pkg/log"
32+
33+
api "github.com/inftyai/manta/api/v1alpha1"
34+
)
35+
36+
// PodReconciler reconciles a Torrent object
37+
type PodReconciler struct {
38+
client.Client
39+
Scheme *runtime.Scheme
40+
}
41+
42+
func NewPodReconciler(client client.Client, scheme *runtime.Scheme) *PodReconciler {
43+
return &PodReconciler{
44+
Client: client,
45+
Scheme: scheme,
46+
}
47+
}
48+
49+
//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
50+
51+
func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
52+
logger := log.FromContext(ctx)
53+
54+
pod := &corev1.Pod{}
55+
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pod); err != nil {
56+
return ctrl.Result{}, client.IgnoreNotFound(err)
57+
}
58+
59+
logger.Info("reconcile Pod")
60+
// This should not happen, double check here.
61+
if pod.Labels == nil || pod.Labels[api.TorrentNameLabelKey] == "" {
62+
return ctrl.Result{}, nil
63+
}
64+
65+
torrentName := pod.Labels[api.TorrentNameLabelKey]
66+
67+
torrent := &api.Torrent{}
68+
if err := r.Get(ctx, types.NamespacedName{Name: torrentName}, torrent); err != nil {
69+
return ctrl.Result{}, err
70+
}
71+
72+
newTorrent := constructTorrent(torrent, pod)
73+
if err := r.Client.Create(ctx, &newTorrent); err != nil {
74+
logger.Error(err, "failed to create Torrent", "Torrent", klog.KObj(&newTorrent))
75+
return ctrl.Result{}, err
76+
}
77+
78+
return ctrl.Result{}, nil
79+
}
80+
81+
func (r *PodReconciler) Create(e event.CreateEvent) bool {
82+
pod, match := e.Object.(*corev1.Pod)
83+
if !match {
84+
return false
85+
}
86+
87+
// Pod should be managed by Manta.
88+
if pod.Labels == nil || pod.Labels[api.TorrentNameLabelKey] == "" {
89+
return false
90+
}
91+
92+
return true
93+
}
94+
95+
func (r *PodReconciler) Update(e event.UpdateEvent) bool {
96+
return false
97+
}
98+
99+
func (r *PodReconciler) Delete(e event.DeleteEvent) bool {
100+
return false
101+
}
102+
103+
func (r *PodReconciler) Generic(e event.GenericEvent) bool {
104+
return false
105+
}
106+
107+
func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error {
108+
return ctrl.NewControllerManagedBy(mgr).
109+
For(&corev1.Pod{}).
110+
WithEventFilter(r).
111+
Complete(r)
112+
}
113+
114+
func constructTorrent(torrent *api.Torrent, pod *corev1.Pod) api.Torrent {
115+
newTorrent := api.Torrent{}
116+
newTorrent.ObjectMeta.Name = torrent.Name + "--tmp--" + pod.Spec.NodeName
117+
newTorrent.TypeMeta = torrent.TypeMeta
118+
newTorrent.Annotations = map[string]string{api.ParentPodNameAnnoKey: pod.Namespace + "/" + pod.Name}
119+
newTorrent.Spec = torrent.Spec
120+
newTorrent.Spec.Preheat = ptr.To[bool](true)
121+
newTorrent.Spec.Replicas = ptr.To[int32](1)
122+
newTorrent.Spec.ReclaimPolicy = ptr.To[api.ReclaimPolicy](api.RetainReclaimPolicy)
123+
newTorrent.Spec.TTLSecondsAfterReady = ptr.To[time.Duration](0)
124+
newTorrent.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": pod.Spec.NodeName}
125+
126+
return newTorrent
127+
}

0 commit comments

Comments
 (0)