Skip to content

Commit

Permalink
improve deployment query performance (#1211)
Browse files Browse the repository at this point in the history 
* added index to deployments
* added page size to visitors
* changed registry get page size to 1k
* fixed missed filter passing in visitors where found
  • Loading branch information
@theganyo
theganyo authored Jul 10, 2023
1 parent 31e01c2 commit 680bdef
Show file tree
Hide file tree
Showing 20 changed files with 135 additions and 114 deletions.
8 changes: 4 additions & 4 deletions cmd/registry/cmd/annotate/annotate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func annotateAPIs(ctx context.Context,
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListAPIs(ctx, client, api, filterFlag, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, api, 0, filterFlag, func(ctx context.Context, api *rpc.Api) error {
taskQueue <- &annotateApiTask{
client: client,
api: api,
Expand All @@ -142,7 +142,7 @@ func annotateVersions(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListVersions(ctx, client, version, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, version, 0, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
taskQueue <- &annotateVersionTask{
client: client,
version: version,
Expand All @@ -159,7 +159,7 @@ func annotateSpecs(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListSpecs(ctx, client, spec, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &annotateSpecTask{
client: client,
spec: spec,
Expand All @@ -176,7 +176,7 @@ func annotateDeployments(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListDeployments(ctx, client, deployment, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
return visitor.ListDeployments(ctx, client, deployment, 0, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
taskQueue <- &annotateDeploymentTask{
client: client,
deployment: deployment,
Expand Down
2 changes: 1 addition & 1 deletion cmd/registry/cmd/check/rules/rule1000/rule1000.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ var requiredArtifacts = &lint.ProjectRule{
}
projectName, _ := names.ParseProject(a.GetName())
var probs []*check.Problem
if err := visitor.ListArtifacts(ctx, client, projectName.Artifact("-"), filter, true, func(ctx context.Context, a *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, projectName.Artifact("-"), 0, filter, true, func(ctx context.Context, a *rpc.Artifact) error {
found[a.GetName()] = true

name, _ := names.ParseArtifact(a.GetName())
Expand Down
4 changes: 2 additions & 2 deletions cmd/registry/cmd/compute/complexity/complexity.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func Command() *cobra.Command {
}

if parsed.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeComplexityTask{
client: client,
specName: spec.Name,
Expand All @@ -74,7 +74,7 @@ func Command() *cobra.Command {
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, parsed, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, parsed, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeComplexityTask{
client: client,
specName: spec.Name,
Expand Down
6 changes: 3 additions & 3 deletions cmd/registry/cmd/compute/conformance/conformance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ func Command() *cobra.Command {

specs := make([]*rpc.ApiSpec, 0)
if name.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, name.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, name.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
specs = append(specs, spec)
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, name, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, name, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
specs = append(specs, spec)
return nil
})
Expand All @@ -76,7 +76,7 @@ func Command() *cobra.Command {
}

guides := make([]*style.StyleGuide, 0)
if err := visitor.ListArtifacts(ctx, client, name.Project().Artifact("-"), styleguideFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, name.Project().Artifact("-"), 0, styleguideFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error {
guide := new(style.StyleGuide)
if err := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), guide); err != nil {
log.FromContext(ctx).WithError(err).Debugf("Unmarshal() to StyleGuide failed on artifact: %s", artifact.GetName())
Expand Down
2 changes: 1 addition & 1 deletion cmd/registry/cmd/compute/lint/lint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func Command() *cobra.Command {
}

// Iterate through a collection of specs and evaluate each.
return visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeLintTask{
client: client,
spec: spec,
Expand Down
14 changes: 7 additions & 7 deletions cmd/registry/cmd/compute/lintstats/lintstats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func computeLintStatsSpecs(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
// Iterate through a collection of specs and evaluate each.
log.Debug(ctx, spec.GetName())
// get the lint results
Expand Down Expand Up @@ -181,10 +181,10 @@ func computeLintStatsProjects(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListProjects(ctx, adminClient, projectName, nil, filter, func(ctx context.Context, project *rpc.Project) error {
return visitor.ListProjects(ctx, adminClient, projectName, nil, 0, filter, func(ctx context.Context, project *rpc.Project) error {
project_stats := &style.LintStats{}

if err := visitor.ListAPIs(ctx, client, projectName.Api(""), filter, func(ctx context.Context, api *rpc.Api) error {
if err := visitor.ListAPIs(ctx, client, projectName.Api(""), 0, filter, func(ctx context.Context, api *rpc.Api) error {
aggregateLintStats(ctx, client, api.GetName(), linter, project_stats)
return nil
}); err != nil {
Expand All @@ -207,10 +207,10 @@ func computeLintStatsAPIs(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListAPIs(ctx, client, apiName, filter, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, apiName, 0, filter, func(ctx context.Context, api *rpc.Api) error {
api_stats := &style.LintStats{}

if err := visitor.ListVersions(ctx, client, apiName.Version(""), filter, func(ctx context.Context, version *rpc.ApiVersion) error {
if err := visitor.ListVersions(ctx, client, apiName.Version(""), 0, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
aggregateLintStats(ctx, client, version.GetName(), linter, api_stats)
return nil
}); err != nil {
Expand All @@ -234,9 +234,9 @@ func computeLintStatsVersions(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListVersions(ctx, client, versionName, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, versionName, 0, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
stats := &style.LintStats{}
if err := visitor.ListSpecs(ctx, client, versionName.Spec(""), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
if err := visitor.ListSpecs(ctx, client, versionName.Spec(""), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
aggregateLintStats(ctx, client, spec.GetName(), linter, stats)
return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/registry/cmd/compute/vocabulary/vocabulary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func Command() *cobra.Command {

// Iterate through a collection of specs and summarize each.
if parsed.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeVocabularyTask{
client: client,
spec: spec,
Expand All @@ -76,7 +76,7 @@ func Command() *cobra.Command {
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, parsed, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, parsed, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeVocabularyTask{
client: client,
spec: spec,
Expand Down
1 change: 1 addition & 0 deletions cmd/registry/cmd/get/get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ func Command() *cobra.Command {
RegistryClient: registryClient,
AdminClient: adminClient,
Pattern: pattern,
PageSize: 1000,
Filter: filter,
}); err != nil {
if status.Code(err) == codes.NotFound {
Expand Down
8 changes: 4 additions & 4 deletions cmd/registry/cmd/label/label.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func labelAPIs(ctx context.Context,
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListAPIs(ctx, client, api, filterFlag, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, api, 0, filterFlag, func(ctx context.Context, api *rpc.Api) error {
taskQueue <- &labelApiTask{
client: client,
api: api,
Expand All @@ -141,7 +141,7 @@ func labelVersions(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListVersions(ctx, client, version, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, version, 0, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
taskQueue <- &labelVersionTask{
client: client,
version: version,
Expand All @@ -158,7 +158,7 @@ func labelSpecs(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListSpecs(ctx, client, spec, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &labelSpecTask{
client: client,
spec: spec,
Expand All @@ -175,7 +175,7 @@ func labelDeployments(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListDeployments(ctx, client, deployment, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
return visitor.ListDeployments(ctx, client, deployment, 0, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
taskQueue <- &labelDeploymentTask{
client: client,
deployment: deployment,
Expand Down
8 changes: 4 additions & 4 deletions cmd/registry/controller/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ type RegistryLister struct {
}

func (r *RegistryLister) ListAPIs(ctx context.Context, api names.Api, filter string, handler visitor.ApiHandler) error {
return visitor.ListAPIs(ctx, r.RegistryClient, api, filter, handler)
return visitor.ListAPIs(ctx, r.RegistryClient, api, 0, filter, handler)
}

func (r *RegistryLister) ListVersions(ctx context.Context, version names.Version, filter string, handler visitor.VersionHandler) error {
return visitor.ListVersions(ctx, r.RegistryClient, version, filter, handler)
return visitor.ListVersions(ctx, r.RegistryClient, version, 0, filter, handler)
}

func (r *RegistryLister) ListSpecs(ctx context.Context, spec names.Spec, filter string, handler visitor.SpecHandler) error {
return visitor.ListSpecs(ctx, r.RegistryClient, spec, filter, false, handler)
return visitor.ListSpecs(ctx, r.RegistryClient, spec, 0, filter, false, handler)
}

func (r *RegistryLister) ListArtifacts(ctx context.Context, artifact names.Artifact, filter string, contents bool, handler visitor.ArtifactHandler) error {
return visitor.ListArtifacts(ctx, r.RegistryClient, artifact, filter, contents, handler)
return visitor.ListArtifacts(ctx, r.RegistryClient, artifact, 0, filter, contents, handler)
}

func listResources(ctx context.Context, client listingClient, pattern, filter string) ([]patterns.ResourceInstance, error) {
Expand Down
6 changes: 3 additions & 3 deletions cmd/registry/patch/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,
var artifacts []*encoding.Artifact
if nested {
versions = make([]*encoding.ApiVersion, 0)
if err = visitor.ListVersions(ctx, client, apiName.Version("-"), "", func(ctx context.Context, message *rpc.ApiVersion) error {
if err = visitor.ListVersions(ctx, client, apiName.Version("-"), 0, "", func(ctx context.Context, message *rpc.ApiVersion) error {
var version *encoding.ApiVersion
version, err := NewApiVersion(ctx, client, message, true)
if err != nil {
Expand All @@ -63,7 +63,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,
return nil, err
}
deployments = make([]*encoding.ApiDeployment, 0)
if err = visitor.ListDeployments(ctx, client, apiName.Deployment("-"), "", func(ctx context.Context, message *rpc.ApiDeployment) error {
if err = visitor.ListDeployments(ctx, client, apiName.Deployment("-"), 0, "", func(ctx context.Context, message *rpc.ApiDeployment) error {
var deployment *encoding.ApiDeployment
deployment, err = NewApiDeployment(ctx, client, message, true)
if err != nil {
Expand Down Expand Up @@ -109,7 +109,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,

func collectChildArtifacts(ctx context.Context, client *gapic.RegistryClient, artifactPattern names.Artifact) ([]*encoding.Artifact, error) {
artifacts := make([]*encoding.Artifact, 0)
if err := visitor.ListArtifacts(ctx, client, artifactPattern, "", true, func(ctx context.Context, message *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, artifactPattern, 0, "", true, func(ctx context.Context, message *rpc.Artifact) error {
artifact, err := NewArtifact(ctx, client, message)
if err != nil {
log.FromContext(ctx).Warnf("Skipping %s: %s", message.Name, err)
Expand Down
Loading

0 comments on commit 680bdef

Please sign in to comment.