-
Notifications
You must be signed in to change notification settings - Fork 3
Testbed Reload Configurations Mechanism #437
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -381,6 +381,7 @@ def __init__(self, port, is_simulated, config): | |
| self.server.register_request_handler('get_service_info', self.on_get_service_info) | ||
| self.server.register_request_handler('register_service', self.on_register_service) | ||
| self.server.register_request_handler('shut_down', self.on_shut_down) | ||
| self.server.register_request_handler('reload_config', self.on_reload_config) | ||
|
|
||
| self.is_running = False | ||
| self.shutdown_requested = threading.Event() | ||
|
|
@@ -668,6 +669,16 @@ def on_register_service(self, data): | |
|
|
||
| return reply.SerializeToString() | ||
|
|
||
| def on_reload_config(self, data): | ||
| request = testbed_proto.ReloadConfigRequest() | ||
| request.ParseFromString(data) | ||
|
|
||
| new_config = json.loads(request.config) | ||
| self.reload_config(new_config) | ||
|
|
||
| reply = testbed_proto.ReloadConfigReply() | ||
| return reply.SerializeToString() | ||
|
|
||
| def on_shut_down(self, data): | ||
| self.shutdown_requested.set() | ||
|
|
||
|
|
@@ -686,6 +697,67 @@ def register_service_type(self, service_type, path): | |
| ''' | ||
| self.service_type_paths[service_type] = path | ||
|
|
||
| def reload_config(self, new_config): | ||
| '''Reload the testbed configuration. | ||
|
|
||
| This updates the configuration without restarting the testbed. | ||
| Services must be restarted to pick up their new configuration. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| new_config : dict | ||
| The new configuration dictionary. | ||
| ''' | ||
| self.log.info('Reloading configuration...') | ||
|
|
||
| # Update the config | ||
| old_config = self.config | ||
| self.config = new_config | ||
|
|
||
| # Update simulation mode if it changed | ||
| if 'testbed' in new_config and 'simulated' in new_config['testbed']: | ||
| self.is_simulated = new_config['testbed']['simulated'] | ||
|
|
||
| # Check for added/removed services | ||
| old_services = set(old_config.get('services', {}).keys()) | ||
| new_services = set(new_config.get('services', {}).keys()) | ||
|
|
||
| added = new_services - old_services | ||
| removed = old_services - new_services | ||
|
|
||
| if added: | ||
| self.log.info(f'New services in config: {list(added)}') | ||
| # Add new service references | ||
| for service_id in added: | ||
| service_info = new_config['services'][service_id] | ||
| service_type = service_info['service_type'] | ||
|
|
||
| if self.is_simulated and 'simulated_service_type' in service_info: | ||
| service_type = service_info['simulated_service_type'] | ||
|
|
||
| dependencies = service_info.get('depends_on', []) | ||
| self.services[service_id] = ServiceReference(service_id, service_type, ServiceState.CLOSED, dependencies, self.message_broker) | ||
|
|
||
| if removed: | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe we could also have a warning for services that are running of which the config has changed?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have been using this mechanism to update flat maps, tip tilt stage position defaults, camera offset_x, offset_y defaults in case we want to start / stop the services and have them come up the the new default without restarting the testbed. Since this is all tracked in the services.yml config file I can track it on git. Generally speaking when I hit reload I want to make sure the new configs are in place. A warning may over alert the user. Maybe info level would be a good way to status what has changed. I could go either way though
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed on |
||
| self.log.warning(f'Services removed from config: {list(removed)}') | ||
| # Check if any removed services are running | ||
| for service_id in removed: | ||
| if service_id in self.services and self.services[service_id].is_alive: | ||
| self.log.warning(f'Service "{service_id}" is running but removed from config. Consider stopping it.') | ||
|
|
||
| # Update startup services list | ||
| self.startup_services = [] | ||
| if 'safety' in self.config.get('testbed', {}): | ||
| self.startup_services.append(self.config['testbed']['safety']['service_id']) | ||
|
|
||
| if 'startup_services' in self.config.get('testbed', {}): | ||
| self.startup_services.extend(self.config['testbed']['startup_services']) | ||
|
|
||
| if self.is_simulated and 'simulator' not in self.startup_services: | ||
| self.startup_services.append('simulator') | ||
|
|
||
| self.log.info('Configuration reloaded successfully.') | ||
|
|
||
| def start_service(self, service_id): | ||
| '''Start a service. | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -197,6 +197,26 @@ void TestbedProxy::ShutDown() | |
| } | ||
| } | ||
|
|
||
| void TestbedProxy::ReloadConfig(const json &new_config) | ||
| { | ||
| catkit_proto::testbed::ReloadConfigRequest request; | ||
| request.set_config(new_config.dump()); | ||
|
|
||
| catkit_proto::testbed::ReloadConfigReply reply; | ||
|
|
||
| try | ||
| { | ||
| reply.ParseFromString(MakeRequest("reload_config", Serialize(request))); | ||
| } | ||
| catch (...) | ||
| { | ||
| throw std::runtime_error("Unable to reload config."); | ||
| } | ||
|
|
||
| // Invalidate cached config so it gets refetched on next access | ||
| m_HasGottenInfo = false; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This variable is just for this instance of a |
||
| } | ||
|
|
||
| std::shared_ptr<DataStream> TestbedProxy::GetHeartbeat() | ||
| { | ||
| GetTestbedInfo(); | ||
|
|
||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This file might've been committed accidentally?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. oops! thanks for catching that |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You're just checking new/removed services, but not updating the dependency graph. A correct dependency graph is required for correct shutdown of the testbed.