Skip to content

Commit

Permalink
Merge pull request #1 from keithmattix/keithmattix/experimental-fields
Browse files Browse the repository at this point in the history 
Experimental Fields Functionality
  • Loading branch information
@whitneygriffith
whitneygriffith authored Feb 20, 2024
2 parents b785c80 + 56ef810 commit e4405c2
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 50 deletions.
73 changes: 50 additions & 23 deletions cmd/protoc-gen-crd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package main

import (
"fmt"
"log"
"strings"

plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
Expand All @@ -25,6 +24,12 @@ import (
"istio.io/tools/pkg/protomodel"
)

const (
standardChannelFileName = "kubernetes/standard.gen.yaml"
experimentalChannelFileName = "kubernetes/experimental.gen.yaml"
legacyChannelFileName = "kubernetes/legacy.gen.yaml"
)

// Breaks the comma-separated list of key=value pairs
// in the parameter string into an easy to use map.
func extractParams(parameter string) map[string]string {
Expand All @@ -47,6 +52,11 @@ func extractParams(parameter string) map[string]string {
func generate(request *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, error) {
includeDescription := true
enumAsIntOrString := false
type genMetadata struct {
shouldGen bool
includeExperimental bool
fds []*protomodel.FileDescriptor
}

p := extractParams(request.GetParameter())
for k, v := range p {
Expand Down Expand Up @@ -74,42 +84,59 @@ func generate(request *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorRespon
}

m := protomodel.NewModel(request, false)

legacyChannelFilesToGen := make(map[*protomodel.FileDescriptor]struct{})
standardChannelFilesToGen := make(map[*protomodel.FileDescriptor]struct{})
experimentalChannelFilesToGen := make(map[*protomodel.FileDescriptor]struct{})
channelOutput := map[string]*genMetadata{
standardChannelFileName: {
shouldGen: true,
includeExperimental: false,
fds: make([]*protomodel.FileDescriptor, 0),
},
experimentalChannelFileName: {
shouldGen: true,
includeExperimental: true,
fds: make([]*protomodel.FileDescriptor, 0),
},
legacyChannelFileName: {
shouldGen: true,
includeExperimental: true,
fds: make([]*protomodel.FileDescriptor, 0),
},
}

for _, fileName := range request.FileToGenerate {
fd := m.AllFilesByName[fileName]
if fd == nil {
return nil, fmt.Errorf("unable to find %s", request.FileToGenerate)
} else if strings.HasSuffix(fd.GetPackage(), "v1") {
standardChannelFilesToGen[fd] = struct{}{}
log.Println("This is a standard channel API: ", fd)
} else if strings.Contains(fd.GetPackage(), "v1alpha") {
// v1alpha1, v1alpha2, etc is considered experimental
experimentalChannelFilesToGen[fd] = struct{}{}
log.Println("This is an experimental channel API: ", fd)
}

// We'll later remove the files from the standard channel that are experimental
channelOutput[standardChannelFileName].fds = append(channelOutput[standardChannelFileName].fds, fd)
channelOutput[experimentalChannelFileName].fds = append(channelOutput[experimentalChannelFileName].fds, fd)
// Legacy channel will have all files that are in standard and experimental
log.Println("This is also added to legacy channel: ", fd)
legacyChannelFilesToGen[fd] = struct{}{}
channelOutput[legacyChannelFileName].fds = append(channelOutput[legacyChannelFileName].fds, fd)
}

descriptionConfiguration := &DescriptionConfiguration{
IncludeDescriptionInSchema: includeDescription,
}

g := newOpenAPIGenerator(
m,
descriptionConfiguration,
enumAsIntOrString)
response := plugin.CodeGeneratorResponse{}
for outputFileName, meta := range channelOutput {
meta := meta
g := newOpenAPIGenerator(
m,
descriptionConfiguration,
enumAsIntOrString,
meta.includeExperimental,
)
filesToGen := map[*protomodel.FileDescriptor]bool{}
for _, fd := range meta.fds {
filesToGen[fd] = meta.shouldGen
}
rf := g.generateSingleFileOutput(filesToGen, outputFileName, meta.includeExperimental)
response.File = append(response.File, &rf)
}

channels := make(map[string]map[*protomodel.FileDescriptor]struct{})
channels["kubernetes/legacy.gen.yaml"] = legacyChannelFilesToGen
channels["kubernetes/experimental.gen.yaml"] = experimentalChannelFilesToGen
channels["kubernetes/standard.gen.yaml"] = standardChannelFilesToGen
return g.generateOutput(channels)
return &response, nil
}

func main() {
Expand Down
76 changes: 49 additions & 27 deletions cmd/protoc-gen-crd/openapiGenerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"golang.org/x/exp/maps"
"google.golang.org/genproto/googleapis/api/annotations"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/pluginpb"
apiextinternal "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
Expand Down Expand Up @@ -122,6 +123,7 @@ type openapiGenerator struct {
descriptionConfiguration *DescriptionConfiguration
enumAsIntOrString bool
customSchemasByMessageName map[string]*apiext.JSONSchemaProps
includeExperimentalFields bool
}

type DescriptionConfiguration struct {
Expand All @@ -133,12 +135,14 @@ func newOpenAPIGenerator(
model *protomodel.Model,
descriptionConfiguration *DescriptionConfiguration,
enumAsIntOrString bool,
includeExperimentalFields bool,
) *openapiGenerator {
return &openapiGenerator{
model: model,
descriptionConfiguration: descriptionConfiguration,
enumAsIntOrString: enumAsIntOrString,
customSchemasByMessageName: buildCustomSchemasByMessageName(),
includeExperimentalFields: includeExperimentalFields,
}
}

Expand All @@ -156,14 +160,6 @@ func buildCustomSchemasByMessageName() map[string]*apiext.JSONSchemaProps {
return schemasByMessageName
}

func (g *openapiGenerator) generateOutput(channels map[string]map[*protomodel.FileDescriptor]struct{}) (*plugin.CodeGeneratorResponse, error) {
response := plugin.CodeGeneratorResponse{}

g.generateSingleFileOutputPerChannel(channels, &response)

return &response, nil
}

func (g *openapiGenerator) getFileContents(
file *protomodel.FileDescriptor,
messages map[string]*protomodel.MessageDescriptor,
Expand All @@ -184,22 +180,22 @@ func (g *openapiGenerator) getFileContents(
}
}

func (g *openapiGenerator) generateSingleFileOutputPerChannel(channels map[string]map[*protomodel.FileDescriptor]struct{}, response *plugin.CodeGeneratorResponse) {
for channel := range channels {
messages := make(map[string]*protomodel.MessageDescriptor)
enums := make(map[string]*protomodel.EnumDescriptor)
descriptions := make(map[string]string)

outputFileName := channel
filesToGen := channels[channel]

log.Println("Generating response file for channel: ", channel)
for file := range filesToGen {
func (g *openapiGenerator) generateSingleFileOutput(
filesToGen map[*protomodel.FileDescriptor]bool,
fileName string,
includeExperimentalFields bool,
) pluginpb.CodeGeneratorResponse_File {
messages := make(map[string]*protomodel.MessageDescriptor)
enums := make(map[string]*protomodel.EnumDescriptor)
descriptions := make(map[string]string)

for file, ok := range filesToGen {
if ok {
g.getFileContents(file, messages, enums, descriptions)
}
rf := g.generateFile(outputFileName, messages, enums, descriptions)
response.File = append(response.File, &rf)
}

return g.generateFile(fileName, messages, enums, descriptions, includeExperimentalFields)
}

const (
Expand Down Expand Up @@ -236,28 +232,35 @@ func cleanComments(lines []string) []string {
return out
}

func parseGenTags(s string) map[string]string {
func parseMessageGenTags(s string) map[string]string {
lines := cleanComments(strings.Split(s, "\n"))
res := map[string]string{}
for _, line := range lines {
if len(line) == 0 {
continue
}
// +cue-gen:AuthorizationPolicy:groupName:security.istio.io turns into
// :AuthorizationPolicy:groupName:security.istio.io
_, contents, f := strings.Cut(line, enableCRDGenTag)
if !f {
continue
}
// :AuthorizationPolicy:groupName:security.istio.io turns into
// ["AuthorizationPolicy", "groupName", "security.istio.io"]
spl := strings.SplitN(contents[1:], ":", 3)
if len(spl) < 2 {
log.Fatalf("invalid tag: %v", line)
log.Fatalf("invalid message tag: %v", line)
}
val := ""
if len(spl) > 2 {
// val = "security.istio.io"
val = spl[2]
}
// res["groupName"] = "security.istio.io;;newVal"
if _, f := res[spl[1]]; f {
res[spl[1]] += ";;" + val
} else {
// res["groupName"] = "security.istio.io"
res[spl[1]] = val
}
}
Expand All @@ -273,22 +276,23 @@ func (g *openapiGenerator) generateFile(
messages map[string]*protomodel.MessageDescriptor,
enums map[string]*protomodel.EnumDescriptor,
descriptions map[string]string,
includeExperimental bool,
) plugin.CodeGeneratorResponse_File {
g.messages = messages

allSchemas := make(map[string]*apiext.JSONSchemaProps)

// Type --> Key --> Value
genTags := map[string]map[string]string{}
messageGenTags := map[string]map[string]string{}

for _, message := range messages {
// we generate the top-level messages here and the nested messages are generated
// inside each top-level message.
if message.Parent == nil {
g.generateMessage(message, allSchemas)
}
if gt := parseGenTags(message.Location().GetLeadingComments()); gt != nil {
genTags[g.absoluteName(message)] = gt
if gt := parseMessageGenTags(message.Location().GetLeadingComments()); gt != nil {
messageGenTags[g.absoluteName(message)] = gt
}
}

Expand All @@ -302,7 +306,11 @@ func (g *openapiGenerator) generateFile(
// Name -> CRD
crds := map[string]*apiext.CustomResourceDefinition{}

for name, cfg := range genTags {
for name, cfg := range messageGenTags {
if cfg["releaseChannel"] == "experimental" && !includeExperimental {
log.Printf("Skipping experimental resource %s for standard channel", name)
continue
}
log.Println("Generating", name)
group := cfg["groupName"]
version := cfg["version"]
Expand Down Expand Up @@ -554,6 +562,9 @@ func (g *openapiGenerator) generateMessageSchema(message *protomodel.MessageDesc
for _, field := range message.Fields {
fn := g.fieldName(field)
sr := g.fieldType(field)
if sr == nil {
continue // This field is skipped for whatever reason; check logs
}
o.Properties[fn] = *sr

if isRequired(field) {
Expand Down Expand Up @@ -695,6 +706,14 @@ func (g *openapiGenerator) generateDescription(desc protomodel.CoreDesc) string
}

func (g *openapiGenerator) fieldType(field *protomodel.FieldDescriptor) *apiext.JSONSchemaProps {
if !g.includeExperimentalFields {
if gt := parseMessageGenTags(field.Location().GetLeadingComments()); gt != nil {
if gt["releaseChannel"] == "experimental" {
log.Println("Skipping experimental field", g.fieldName(field), "for standard channel")
return nil
}
}
}
schema := &apiext.JSONSchemaProps{}
var isMap bool
switch *field.Type {
Expand Down Expand Up @@ -740,6 +759,9 @@ func (g *openapiGenerator) fieldType(field *protomodel.FieldDescriptor) *apiext.
} else if msg.GetOptions().GetMapEntry() {
isMap = true
sr := g.fieldType(msg.Fields[1])
if sr == nil {
return nil
}
schema = sr
schema = &apiext.JSONSchemaProps{
Type: "object",
Expand Down

0 comments on commit e4405c2

Please sign in to comment.