ReloadSchema{Shard,Keyspace} and UI fixes.

- Adding vtctl ReloadSchema{Shard,Keyspace} commands.
- Adding ReloadSchema{Shard,Keyspace} to ui.
- Removing test workflow data to avoid flickering at startup.
- Fixing schema display in vtctld2 UI.
- Rebuild vtctl doc.
- Rebuilding UI.

Author: alainjobart

doc/vtctlReference.md

@@ -1073,6 +1073,8 @@ Updates the configuration of the MaxReplicationLag module. The configuration mus
 * [GetVSchema](#getvschema)
 * [RebuildVSchemaGraph](#rebuildvschemagraph)
 * [ReloadSchema](#reloadschema)
+* [ReloadSchemaKeyspace](#reloadschemakeyspace)
+* [ReloadSchemaShard](#reloadschemashard)
 * [ValidatePermissionsKeyspace](#validatepermissionskeyspace)
 * [ValidatePermissionsShard](#validatepermissionsshard)
 * [ValidateSchemaKeyspace](#validateschemakeyspace)
@@ -1261,6 +1263,56 @@ Reloads the schema on a remote tablet.
 * the <code>&lt;tablet alias&gt;</code> argument is required for the <code>&lt;ReloadSchema&gt;</code> command This error occurs if the command is not called with exactly one argument.
 
 
+### ReloadSchemaKeyspace
+
+Reloads the schema on all the tablets in a keyspace.
+
+#### Example
+
+<pre class="command-example">ReloadSchemaKeyspace [-concurrency=10] [-include_master=false] &lt;keyspace&gt;</pre>
+
+#### Flags
+
+| Name | Type | Definition |
+| :-------- | :--------- | :--------- |
+| concurrency | Int | How many tablets to reload in parallel |
+| include_master | Boolean | Include the master tablet(s) |
+
+
+#### Arguments
+
+* <code>&lt;keyspace&gt;</code> &ndash; Required. The name of a sharded database that contains one or more tables. Vitess distributes keyspace shards into multiple machines and provides an SQL interface to query the data. The argument value must be a string that does not contain whitespace.
+
+#### Errors
+
+* the <code>&lt;keyspace&gt;</code> argument is required for the <code>&lt;ReloadSchemaKeyspace&gt;</code> command This error occurs if the command is not called with exactly one argument.
+
+
+### ReloadSchemaShard
+
+Reloads the schema on all the tablets in a shard.
+
+#### Example
+
+<pre class="command-example">ReloadSchemaShard [-concurrency=10] [-include_master=false] &lt;keyspace/shard&gt;</pre>
+
+#### Flags
+
+| Name | Type | Definition |
+| :-------- | :--------- | :--------- |
+| concurrency | Int | How many tablets to reload in parallel |
+| include_master | Boolean | Include the master tablet |
+
+
+#### Arguments
+
+* <code>&lt;keyspace/shard&gt;</code> &ndash; Required. The name of a sharded database that contains one or more tables as well as the shard associated with the command. The keyspace must be identified by a string that does not contain whitepace, while the shard is typically identified by a string in the format <code>&lt;range start&gt;-&lt;range end&gt;</code>.
+
+#### Errors
+
+* the <code>&lt;keyspace/shard&gt;</code> argument is required for the <code>&lt;ReloadSchemaShard&gt;</code> command This error occurs if the command is not called with exactly one argument.
+
+
 ### ValidatePermissionsKeyspace
 
 Validates that the master permissions from shard 0 match those of all of the other tablets in the keyspace.

go/vt/schemamanager/tablet_executor.go

@@ -11,6 +11,7 @@ import (
 
 	"golang.org/x/net/context"
 
+	"github.com/youtube/vitess/go/sync2"
 	"github.com/youtube/vitess/go/vt/mysqlctl/tmutils"
 	"github.com/youtube/vitess/go/vt/sqlparser"
 	"github.com/youtube/vitess/go/vt/wrangler"
@@ -179,7 +180,7 @@ func (exec *TabletExecutor) preflightSchemaChanges(ctx context.Context, sqls []s
 				// DROP IF EXISTS on a nonexistent table does not change the schema. It's safe to ignore.
 				continue
 			}
-			return fmt.Errorf("Schema change: '%s' does not introduce any table definition change.", sqls[i])
+			return fmt.Errorf("schema change: '%s' does not introduce any table definition change", sqls[i])
 		}
 	}
 	exec.schemaDiffs = schemaDiffs
@@ -261,13 +262,14 @@ func (exec *TabletExecutor) executeOnAllTablets(ctx context.Context, execResult
 	// If all shards succeeded, wait (up to waitSlaveTimeout) for slaves to
 	// execute the schema change via replication. This is best-effort, meaning
 	// we still return overall success if the timeout expires.
+	concurrency := sync2.NewSemaphore(10, 0)
 	reloadCtx, cancel := context.WithTimeout(ctx, exec.waitSlaveTimeout)
 	defer cancel()
 	for _, result := range execResult.SuccessShards {
 		wg.Add(1)
 		go func(result ShardResult) {
 			defer wg.Done()
-			exec.wr.ReloadSchemaShard(reloadCtx, exec.keyspace, result.Shard, result.Position)
+			exec.wr.ReloadSchemaShard(reloadCtx, exec.keyspace, result.Shard, result.Position, concurrency, false /* includeMaster */)
 		}(result)
 	}
 	wg.Wait()

go/vt/vtctl/vtctl.go

@@ -95,6 +95,7 @@ import (
 	"github.com/youtube/vitess/go/flagutil"
 	"github.com/youtube/vitess/go/mysqlconn/replication"
 	"github.com/youtube/vitess/go/sqltypes"
+	"github.com/youtube/vitess/go/sync2"
 	hk "github.com/youtube/vitess/go/vt/hook"
 	"github.com/youtube/vitess/go/vt/key"
 	"github.com/youtube/vitess/go/vt/logutil"
@@ -323,6 +324,12 @@ var commands = []commandGroup{
 			{"ReloadSchema", commandReloadSchema,
 				"<tablet alias>",
 				"Reloads the schema on a remote tablet."},
+			{"ReloadSchemaShard", commandReloadSchemaShard,
+				"[-concurrency=10] [-include_master=false] <keyspace/shard>",
+				"Reloads the schema on all the tablets in a shard."},
+			{"ReloadSchemaKeyspace", commandReloadSchemaKeyspace,
+				"[-concurrency=10] [-include_master=false] <keyspace>",
+				"Reloads the schema on all the tablets in a keyspace."},
 			{"ValidateSchemaShard", commandValidateSchemaShard,
 				"[-exclude_tables=''] [-include-views] <keyspace/shard>",
 				"Validates that the master schema matches all of the slaves."},
@@ -1785,6 +1792,37 @@ func commandReloadSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *f
 	return wr.ReloadSchema(ctx, tabletAlias)
 }
 
+func commandReloadSchemaShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
+	concurrency := subFlags.Int("concurrency", 10, "How many tablets to reload in parallel")
+	includeMaster := subFlags.Bool("include_master", true, "Include the master tablet")
+	if err := subFlags.Parse(args); err != nil {
+		return err
+	}
+	if subFlags.NArg() != 1 {
+		return fmt.Errorf("the <keyspace/shard> argument is required for the ReloadSchemaShard command")
+	}
+	keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
+	if err != nil {
+		return err
+	}
+	sema := sync2.NewSemaphore(*concurrency, 0)
+	wr.ReloadSchemaShard(ctx, keyspace, shard, "" /* waitPosition */, sema, *includeMaster)
+	return nil
+}
+
+func commandReloadSchemaKeyspace(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
+	concurrency := subFlags.Int("concurrency", 10, "How many tablets to reload in parallel")
+	includeMaster := subFlags.Bool("include_master", true, "Include the master tablet(s)")
+	if err := subFlags.Parse(args); err != nil {
+		return err
+	}
+	if subFlags.NArg() != 1 {
+		return fmt.Errorf("the <keyspace> argument is required for the ReloadSchemaKeyspace command")
+	}
+	sema := sync2.NewSemaphore(*concurrency, 0)
+	return wr.ReloadSchemaKeyspace(ctx, subFlags.Arg(0), sema, *includeMaster)
+}
+
 func commandValidateSchemaShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
 	excludeTables := subFlags.String("exclude_tables", "", "Specifies a comma-separated list of tables to exclude. Each is either an exact match, or a regular expression of the form /regexp/")
 	includeViews := subFlags.Bool("include-views", false, "Includes views in the validation")

go/vt/wrangler/schema.go

@@ -16,6 +16,7 @@ import (
 
 	log "github.com/golang/glog"
 	"github.com/youtube/vitess/go/sqltypes"
+	"github.com/youtube/vitess/go/sync2"
 	"github.com/youtube/vitess/go/vt/concurrency"
 	"github.com/youtube/vitess/go/vt/mysqlctl/tmutils"
 	"github.com/youtube/vitess/go/vt/topo"
@@ -56,31 +57,35 @@ func (wr *Wrangler) ReloadSchema(ctx context.Context, tabletAlias *topodatapb.Ta
 // and the periodic schema reload makes them self-healing anyway.
 // So we do this on a best-effort basis, and log warnings for any tablets
 // that fail to reload within the context deadline.
-// Note that this skips the master because it's assumed that the schema
-// was reloaded there right after applying it.
-func (wr *Wrangler) ReloadSchemaShard(ctx context.Context, keyspace, shard, replicationPos string) {
+func (wr *Wrangler) ReloadSchemaShard(ctx context.Context, keyspace, shard, replicationPos string, concurrency *sync2.Semaphore, includeMaster bool) {
 	tablets, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard)
-	if err != nil {
-		if err != topo.ErrPartialResult {
-			// This is best-effort, so just log it and move on.
-			wr.logger.Warningf("Failed to reload schema on slave tablets in %v/%v (use vtctl ReloadSchema to fix individual tablets): %v", keyspace, shard, err)
-			return
-		}
-		// We got a partial result. Do what we can, but warn that some may be missed.
-		wr.logger.Warningf("Failed to fetch all tablets for %v/%v. Some slave tablets may not have schema reloaded (use vtctl ReloadSchema to fix individual tablets)", keyspace, shard)
+	switch err {
+	case topo.ErrPartialResult:
+		// We got a partial result. Do what we can, but warn
+		// that some may be missed.
+		wr.logger.Warningf("ReloadSchemaShard(%v/%v) got a partial tablet list. Some tablets may not have schema reloaded (use vtctl ReloadSchema to fix individual tablets)", keyspace, shard)
+	case nil:
+		// Good case, keep going too.
+	default:
+		// This is best-effort, so just log it and move on.
+		wr.logger.Warningf("ReloadSchemaShard(%v/%v) failed to load tablet list, will not reload schema (use vtctl ReloadSchemaShard to try again): %v", keyspace, shard, err)
+		return
 	}
 
 	var wg sync.WaitGroup
 	for _, ti := range tablets {
-		if ti.Type == topodatapb.TabletType_MASTER {
-			// We don't need to reload on the master because we assume
-			// ExecuteFetchAsDba() already did that.
+		if !includeMaster && ti.Type == topodatapb.TabletType_MASTER {
+			// We don't need to reload on the master
+			// because we assume ExecuteFetchAsDba()
+			// already did that.
 			continue
 		}
 
 		wg.Add(1)
 		go func(tablet *topodatapb.Tablet) {
 			defer wg.Done()
+			concurrency.Acquire()
+			defer concurrency.Release()
 			if err := wr.tmc.ReloadSchema(ctx, tablet, replicationPos); err != nil {
 				wr.logger.Warningf(
 					"Failed to reload schema on slave tablet %v in %v/%v (use vtctl ReloadSchema to try again): %v",
@@ -91,6 +96,21 @@ func (wr *Wrangler) ReloadSchemaShard(ctx context.Context, keyspace, shard, repl
 	wg.Wait()
 }
 
+// ReloadSchemaKeyspace reloads the schema in all shards in a
+// keyspace.  The concurrency is shared across all shards (only that
+// many tablets will be reloaded at once).
+func (wr *Wrangler) ReloadSchemaKeyspace(ctx context.Context, keyspace string, concurrency *sync2.Semaphore, includeMaster bool) error {
+	shards, err := wr.ts.GetShardNames(ctx, keyspace)
+	if err != nil {
+		return err
+	}
+
+	for _, shard := range shards {
+		wr.ReloadSchemaShard(ctx, keyspace, shard, "" /* waitPosition */, concurrency, includeMaster)
+	}
+	return nil
+}
+
 // helper method to asynchronously diff a schema
 func (wr *Wrangler) diffSchema(ctx context.Context, masterSchema *tabletmanagerdatapb.SchemaDefinition, masterTabletAlias, alias *topodatapb.TabletAlias, excludeTables []string, includeViews bool, wg *sync.WaitGroup, er concurrency.ErrorRecorder) {
 	defer wg.Done()
@@ -321,9 +341,10 @@ func (wr *Wrangler) CopySchemaShard(ctx context.Context, sourceTabletAlias *topo
 	}
 
 	// Notify slaves to reload schema. This is best-effort.
+	concurrency := sync2.NewSemaphore(10, 0)
 	reloadCtx, cancel := context.WithTimeout(ctx, waitSlaveTimeout)
 	defer cancel()
-	wr.ReloadSchemaShard(reloadCtx, destKeyspace, destShard, destMasterPos)
+	wr.ReloadSchemaShard(reloadCtx, destKeyspace, destShard, destMasterPos, concurrency, true /* includeMaster */)
 	return nil
 }
 

test/cache_invalidation.py

@@ -84,8 +84,7 @@ def setUpModule():
     master_tablet.mquery('vt_test_keyspace', _create_vt_a)
     master_tablet.mquery('vt_test_keyspace', _create_vt_b)
 
-    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
-    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
+    utils.run_vtctl(['ReloadSchemaShard', 'test_keyspace/0'])
     utils.run_vtctl(['RebuildVSchemaGraph'])
 
     utils.VtGate().start(tablets=[master_tablet, replica_tablet])

test/update_stream.py

@@ -97,8 +97,7 @@ def setUpModule():
     master_tablet.mquery('vt_test_keyspace', _create_vt_a)
     master_tablet.mquery('vt_test_keyspace', _create_vt_b)
 
-    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
-    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
+    utils.run_vtctl(['ReloadSchemaKeyspace', 'test_keyspace'])
     utils.run_vtctl(['RebuildVSchemaGraph'])
 
     utils.VtGate().start(tablets=[master_tablet, replica_tablet])

web/vtctld2/app/index.html

@@ -27,5 +27,5 @@
 </head>
 <body class="flex-column">
   <vt-app-root class="flex-column flex-grow">Loading...</vt-app-root>
-<script type="text/javascript" src="inline.js"></script><script type="text/javascript" src="styles.07f8743f5392cfdfbcb5.bundle.js"></script><script type="text/javascript" src="main.7e82c61ecfeb791ebb7e.bundle.js"></script></body>
+<script type="text/javascript" src="inline.js"></script><script type="text/javascript" src="styles.07f8743f5392cfdfbcb5.bundle.js"></script><script type="text/javascript" src="main.bd7c386e52651518a1ce.bundle.js"></script></body>
 </html>

web/vtctld2/app/inline.js

@@ -8,7 +8,8 @@ import { DialogSettings } from '../shared/dialog/dialog-settings';
 import { NewShardFlags } from '../shared/flags/shard.flags';
 import { KeyspaceService } from '../api/keyspace.service';
 import { PrepareResponse } from '../shared/prepare-response';
-import { RebuildKeyspaceGraphFlags, RemoveKeyspaceCellFlags, ValidateKeyspaceFlags,
+import { RebuildKeyspaceGraphFlags, ReloadSchemaKeyspaceFlags,
+         RemoveKeyspaceCellFlags, ValidateKeyspaceFlags,
          ValidateSchemaFlags, ValidateVersionFlags } from '../shared/flags/keyspace.flags';
 import { VtctlService } from '../api/vtctl.service';
 
@@ -44,8 +45,9 @@ export class KeyspaceComponent implements OnInit, OnDestroy {
         {label: 'Validate Schema', command: (event) => {this.openValidateSchemaDialog(); }},
         {label: 'Validate Version', command: (event) => {this.openValidateVersionDialog(); }},
       ]},
-      {label: 'Rebuild', items: [
+      {label: 'Rebuild / Reload', items: [
         {label: 'Rebuild Keyspace Graph', command: (event) => {this.openRebuildKeyspaceGraphDialog(); }},
+        {label: 'Reload Schema in Keyspace', command: (event) => {this.openReloadSchemaKeyspaceDialog(); }},
       ]},
       {label: 'Change', items: [
         {label: 'Remove Keyspace Cell', command: (event) => {this.openRemoveKeyspaceCellDialog(); }},
@@ -143,6 +145,15 @@ export class KeyspaceComponent implements OnInit, OnDestroy {
     this.dialogSettings.toggleModal();
   }
 
+  openReloadSchemaKeyspaceDialog() {
+    this.dialogSettings = new DialogSettings('Reload Schema', `Reload Schema in ${this.keyspaceName}`, '',
+                                             'There was a problem reloading schema in {{keyspace_name}}:');
+    this.dialogSettings.setMessage('Reloaded {{keyspace_name}} Schema');
+    let flags = new ReloadSchemaKeyspaceFlags(this.keyspaceName).flags;
+    this.dialogContent = new DialogContent('keyspace_name', flags, {}, undefined, 'ReloadSchemaKeyspace');
+    this.dialogSettings.toggleModal();
+  }
+
   openRemoveKeyspaceCellDialog() {
     this.dialogSettings = new DialogSettings('Remove', `Remove a cell from ${this.keyspaceName}`, '',
                                              'There was a problem removing {{cell_name}}:');