diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml index 202f0a550b1de..659cd2fd73e43 100644 --- a/config/_default/menus/main.en.yaml +++ b/config/_default/menus/main.en.yaml @@ -130,65 +130,65 @@ menu: url: getting_started/integrations/terraform/ weight: 1204 parent: getting_started_integrations + - name: Internal Developer Portal + identifier: getting_started_internal_developer_portal + url: getting_started/internal_developer_portal/ + parent: getting_started + weight: 13 - name: Logs identifier: getting_started_logs url: getting_started/logs/ parent: getting_started - weight: 13 + weight: 14 - name: Monitors identifier: getting_started_monitors url: getting_started/monitors/ parent: getting_started - weight: 14 + weight: 15 - name: OpenTelemetry identifier: getting_started_opentelemetry url: getting_started/opentelemetry/ parent: getting_started - weight: 15 + weight: 16 - name: Profiler identifier: getting_started_profiler url: getting_started/profiler/ parent: getting_started - weight: 16 + weight: 17 - name: Session Replay identifier: getting_started_session_replay url: getting_started/session_replay/ parent: getting_started - weight: 17 + weight: 18 - name: Security identifier: getting_started_security url: getting_started/security/ parent: getting_started - weight: 18 + weight: 19 - name: App and API Protection identifier: getting_started_application_security url: getting_started/security/application_security parent: getting_started_security - weight: 1801 + weight: 1901 - name: Cloud Security identifier: getting_started_cloud_security_management url: getting_started/security/cloud_security_management/ parent: getting_started_security - weight: 1802 + weight: 1902 - name: Cloud SIEM identifier: getting_started_cloud_siem url: getting_started/security/cloud_siem/ parent: getting_started_security - weight: 1803 + weight: 1903 - name: Code Security identifier: getting_started_code_security url: getting_started/code_security/ parent: getting_started_security - weight: 1804 + weight: 1904 - name: Serverless for AWS Lambda identifier: getting_started_serverless url: getting_started/serverless/ parent: getting_started - weight: 19 - - name: Software Catalog - identifier: getting_started_software_catalog - url: getting_started/software_catalog/ - parent: getting_started weight: 20 - name: Software Delivery identifier: getting_started_software_delivery diff --git a/content/en/getting_started/internal_developer_portal/_index.md b/content/en/getting_started/internal_developer_portal/_index.md new file mode 100644 index 0000000000000..135e1d3ae22af --- /dev/null +++ b/content/en/getting_started/internal_developer_portal/_index.md @@ -0,0 +1,196 @@ +--- +title: Getting Started with Internal Developer Portal +aliases: + - /getting_started/service_catalog + - /getting_started/software_catalog/ +further_reading: + - link: '/internal_developer_portal' + tag: 'Documentation' + text: 'Internal Developer Portal' + - link: '/internal_developer_portal/software_catalog/' + tag: 'Documentation' + text: 'Software Catalog' + - link: 'https://learn.datadoghq.com/courses/managing-service-catalog' + tag: 'Learning Center' + text: 'Managing Services with the Service Catalog' + - link: 'https://www.datadoghq.com/blog/service-owner-knowledge-with-datadog-service-catalog/' + tag: 'Blog' + text: 'Simplify microservice governance with the Datadog Service Catalog' + - link: '/internal_developer_portal/scorecards' + tag: 'Documentation' + text: 'Scorecards' + - link: '/internal_developer_portal/self_service_actions' + tag: 'Documentation' + text: 'Self-Service Actions' + - link: '/internal_developer_portal/eng_reports' + tag: 'Documentation' + text: 'Engineering Reports' + - link: '/internal_developer_portal/overview_pages' + tag: 'Documentation' + text: 'Overview Pages' +--- + +{{< img src="tracing/internal_developer_portal/scrolling_the_catalog.mp4" alt="A video that scrolls through the Internal Developer Portal Catalog page and clicks on a service to show a dependency graph with parent and child services represented" video=true >}} + +## Overview + +Datadog's Internal Developer Portal (IDP) helps you unify software metadata, live telemetry, and developer workflows in a single platform. This guide walks you through setting up each core IDP component: + +- [Software Catalog][1]: Real-time inventory of entities and environments, enriched with ownership and operational metadata. +- [Scorecards][2]: Measure adoption of engineering best practices using rules-based evaluations. +- [Self-Service Actions][3]: Enable developers to execute standardized tasks with a single click. +- [Engineering Reports][4]: Visualize quality, reliability, and compliance metrics across your stack. +- [Overview Pages][5]: Give developers and teams a personalized view of their entities, issues, and action items. + +Whether you're starting from scratch or integrating with existing systems like Backstage or ServiceNow, use this guide to get started with IDP. + + +## Prerequisites + +If you have not already, create a [Datadog account][6]. + +## Step 1: Populate Software Catalog + +IDP starts with [Software Catalog][1], a real-time inventory of your software architecture's key building blocks. In Datadog, these are called entities—they can represent individual services, APIs, or grouped Systems. + +You can add entities from: + +- [APM, USM, and RUM][7]: Automatically discovered based on application telemetry +- [Manual definitions][8]: Created through [Datadog][9] or imported through tools like Terraform, Datadog APIs, or the GitHub integration +- [Third-party systems][10]: Through integrations with ServiceNow and Backstage + +Start with a basic definition to register the entity, then enrich it with metadata to add ownership and operational context, including: +- Ownership info and team contacts +- Documentation, dashboards, runbooks +- Deployment pipelines and config links +- Production readiness data through unified service tagging + +The following example defines a `system` entity representing an application composed of multiple services. It includes metadata such as display name, ownership, contacts, related documentation, integrations, and associated service components. + +{{< code-block lang="yaml" filename="entity.datadog.yaml" disable_copy="true" collapsible="true" >}} +apiVersion: v3 + kind: system + metadata: + name: myapp + displayName: My App + tags: + - tag:value + links: + - name: shopping-cart runbook + type: runbook + url: https://runbook/shopping-cart + - name: shopping-cart architecture + provider: gdoc + url: https://google.drive/shopping-cart-architecture + type: doc + - name: shopping-cart Wiki + provider: wiki + url: https://wiki/shopping-cart + type: doc + - name: shopping-cart source code + provider: github + url: http://github/shopping-cart + type: repo + contacts: + - name: Support Slack + type: slack + contact: https://www.slack.com/archives/shopping-cart + owner: myteam + additionalOwners: + - name: opsTeam + type: operator + integrations: + pagerduty: + serviceURL: https://www.pagerduty.com/service-directory/Pshopping-cart + spec: + components: + - service:myservice + - service:otherservice +{{< /code-block >}} + +Read the [Software Catalog setup guide][11] to learn how to add or import entities, and review the [entity model reference][12] for schema details. + +## Step 2: Evaluate entity quality with Scorecards + +Use Scorecards to assess whether entities meet your organization's standards. Scorecards can measure: +- Monitoring coverage +- Production readiness +- Security posture +- Internal tooling adoption +- Ownership and documentation + +Datadog Scorecards include 10 out-of-the-box rules across observability practices, ownership tagging, and production readiness checkpoints. You can group rules into levels to categorize them by their criticality-level 1 (basic expectations), level 2 (recommended practices), and level 3 (advanced/aspirational goals). + +In addition to using default rules, you can define custom rules to reflect your internal standards: + +1. Go to the [Scorecards page][13] and click Create Rule. +1. Specify the rule name, the scorecard it belongs to, a description, and the owning team. +1. Send an outcome of `pass`, `fail`, or `skip` for each entity in one of the following ways: + - Manually through the Datadog UI + - Programmatically through the Scorecards API + - Automatically, using [Workflow Automation][14] to post outcomes on a schedule +1. View an overview of outcomes on the [Scorecards page][13]. + +Learn more about scorecard configuration and custom rules in the [Scorecards documentation][2]. + +## Step 3: Use Self-Service Actions + +Self-Service Actions let you run repeatable tasks through a UI or API. For example, use Self-Service Actions to: +- [Create an S3 bucket with Terraform][16] +- [Scaffold a new project in GitHub][17] +- [Manage Kubernetes deployments][18] + +Actions can be backed by automation systems like Terraform, GitHub Actions, or internal scripts. Self Service Actions offer over 1000+ pre-built integrations with tools across source code management (for example, GitHub and GitLab), ticketing and incident management (for example, Jira, ServiceNow, and PagerDuty), chat (for example, Slack and Microsoft Teams), cloud providers (for example, AWS, GCP, and Azure), and more. You can connect with any additional endpoints, including private resources, leveraging HTTP requests and private action runners. + +Get started by exploring the Self-Service Actions [Blueprint Library][19] in Datadog for example apps that you can customize for your use case. + +To start automating developer workflows, explore the [Self-Service Actions documentation][3]. + +## Step 4: Monitor engineering health with reports + +Engineering Reports provide an at-a-glance view of: +- Scorecards performance across all teams +- Org-wide reliability trends based on SLOs and incident performance +- Velocity and stability of software development + +Explore [Engineering Reports][20] in the Datadog app. These reports are automatically generated and updated in real time. + +Read the [Engineering Reports documentation][4] for more details on available reports and configuration options. + +## Step 5: Use overview pages for personalized insights + +Overview pages surface high-level metrics and action items tailored to individual contributors and teams. + +Start with the [developer overview page][21], which shows: +- Your open Jira tickets and GitHub PRs +- Your team's Monitors, Incidents, SLOs, and Scorecards +- Active issues, errors, and Watchdog alerts + +For setup and customization tips, read the [Overview Pages documentation][22]. + + +## Further reading + +{{< partial name="whats-next/whats-next.html" >}} + +[1]: /internal_developer_portal/software_catalog/ +[2]: /internal_developer_portal/scorecards/ +[3]: /internal_developer_portal/self_service_actions +[4]: /internal_developer_portal/eng_reports +[5]: /internal_developer_portal/overview_pages +[6]: https://www.datadoghq.com/ +[7]: /internal_developer_portal/software_catalog/set_up/discover_entities/ +[8]: /internal_developer_portal/software_catalog/set_up/create_entities/ +[9]: https://app.datadoghq.com/software +[10]: /internal_developer_portal/software_catalog/set_up/import_entities/ +[11]: /internal_developer_portal/software_catalog/set_up/ +[12]: /internal_developer_portal/software_catalog/entity_model/entity_types/ +[13]: https://app.datadoghq.com/software/scorecards +[14]: https://app.datadoghq.com/workflow/blueprints?selected_category=SCORECARDS +[16]: https://app.datadoghq.com/app-builder/apps/edit?startModalOpen=false&template=create-new-s3-bucket&viewMode=templatePreview +[17]: https://app.datadoghq.com/app-builder/apps/edit?startModalOpen=false&template=scaffold-github&viewMode=templatePreview +[18]: https://app.datadoghq.com/app-builder/apps/edit?startModalOpen=false&template=manage-kubernetes-deployments&viewMode=templatePreview +[19]: https://app.datadoghq.com/software/self-service-actions +[20]: https://app.datadoghq.com/idp/reports +[21]: https://app.datadoghq.com/idp/overview/developer +[22]: /internal_developer_portal/overview_pages/ \ No newline at end of file diff --git a/content/en/getting_started/software_catalog/_index.md b/content/en/getting_started/software_catalog/_index.md deleted file mode 100644 index 05c0fa602ead8..0000000000000 --- a/content/en/getting_started/software_catalog/_index.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -title: Getting Started with Software Catalog -aliases: - - /getting_started/service_catalog -further_reading: - - link: '/software_catalog/' - tag: 'Documentation' - text: 'Software Catalog' - - link: 'https://learn.datadoghq.com/courses/managing-service-catalog' - tag: 'Learning Center' - text: 'Managing Services with the Service Catalog' - - link: 'https://www.datadoghq.com/blog/service-owner-knowledge-with-datadog-service-catalog/' - tag: 'Blog' - text: 'Simplify microservice governance with the Datadog Service Catalog' - - link: '/software_catalog/troubleshooting' - tag: 'Documentation' - text: 'Troubleshooting' - - link: '/software_catalog/scorecards' - tag: 'Documentation' - text: 'Service Scorecards' ---- - -{{< img src="/getting_started/software_catalog/overview_image.jpeg" alt="Software Catalog Reliability view showing several services and their associated MTTR, deployment metrics, issues, incidents, SLOs, and monitor statuses." style="width:90%;" >}} - -## Overview - -Datadog Software Catalog provides a consolidated view of your services, combining ownership metadata, performance insights, security analysis, cost allocation, and much more. Having this centralized hub allows your development teams to discover and manage critical components in your runtime environments. - -This page walks you through getting started with Software Catalog in Datadog. - -## Prerequisites - -If you have not already, create a [Datadog account][1]. - -## Adding entries to Software Catalog - -### Automatically detected services - -Software Catalog automatically discovers services based on application performance telemetries such as [APM][2], [USM][3], and [RUM][4]. The integration with APM enables Datadog to routinely discover new services at the same frequency as your traces are collected. With USM, the Datadog Agent connects to your eBPF-compatible hosts. USM automatically detects the services running on this infrastructure and tags them using [unified service tagging][5]. - -### User-defined services - -If you are not using these products, you can manually declare services as entries in the `service.definition.yaml` registry. The definitions in this file include all the metadata that the catalog uses to file your services. These can be created and updated programmatically using an internal API or with a configuration management service like Terraform. You should include this file in your version control and regularly update it whenever new resources are added to your environment. - -The following example represents a `shopping-cart` service from an e-commerce application. It includes important metadata about the service, such as its owning team, languages used, runbook link, and code repository. - -{{< code-block lang="yaml" filename="service.datadog.yaml" collapsible="true" >}} -schema-version: v2.2 -dd-service: shopping-cart -team: e-commerce -application: shopping-app -tier: "1" -type: web -languages: - - go - - python -contacts: - - type: slack - contact: https://yourorg.slack.com/archives/e-commerce - - type: email - contact: ecommerce@example.com - - type: microsoft-teams - contact: https://teams.microsoft.com/example -links: - - name: Runbook - type: runbook - url: http://runbook/shopping-cart - - name: Source - type: repo - provider: github - url: https://github.com/shopping-cart - - name: Deployment - type: repo - provider: github - url: https://github.com/shopping-cart - - name: Config - type: repo - provider: github - url: https://github.com/consul-config/shopping-cart - - name: E-Commerce Team - type: doc - provider: wiki - url: https://wiki/ecommerce - - name: Shopping Cart Architecture - type: doc - provider: wiki - url: https://wiki/ecommerce/shopping-cart - - name: Shopping Cart RFC - type: doc - provider: google doc - url: https://doc.google.com/shopping-cart -tags: - - business-unit:retail - - cost-center:engineering -integrations: - pagerduty: - service-url: https://www.pagerduty.com/service-directory/PSHOPPINGCART - opsgenie: - service-url: "https://www.opsgenie.com/service/uuid" - region: "US" -ci-pipeline-fingerprints: - - id1 - - id2 -extensions: - additionalProperties: - customField1: customValue1 - customField2: customValue2 -{{< /code-block >}} - -You can also use any existing knowledge sources your organization maintains, such as Configuration Management Database (CMDB) tables (through [Datadog's ServiceNow integration][6]) and spreadsheets, to populate your Software Catalog. Datadog also has an [integration with Backstage][7] that allows you to import any data or services registered here into Datadog directly. - -Finally, you can also create entries from `service` tags in other Datadog products like Infrastructure Monitoring and Log Management. - -{{< img src="/getting_started/software_catalog/import_entries.jpeg" alt="Import Entries tab in the Software Catalog setup and configuration section" style="width:90%;" >}} - -## Managing metadata in Software Catalog - -After you create these initial entries in your Software Catalog, it is important to keep the catalog updated consistently so that it remains effective. Service definition files should exist within your team's version control to make it easier to reference new deployments and other changes to the services where an update may be required. - -You can also automate the management of your service metadata by configuring a [GitHub action][8]. This will also allow you to ensure that teams are declaring services in a way that meets your standards, such as requiring all of your production services to have valid runbook links. - -If your organization has an existing registry of ownership, including internal systems like [Backstage][9] or a spreadsheet, a central team can schedule updates to service definition files using [API calls][10]. - -### Connect telemetry from across your monitoring stack - -Connect monitoring data from the rest of your observability platform to improve the utility of your catalog. - -With [unified service tagging][5], you can use the `service` tag to cross-reference service entities in the Software Catalog across all Datadog products. These tags can help enrich your service entities with metadata, metrics, and other context sources like Infrastructure Monitoring, RUM, Log Management, Software Delivery, and Security. - -Application performance telemetry from Universal Service Monitoring and APM also provide out-of-the-box dependency mapping for your system ecosystem, so you can see how components interact with each other in all your runtime environments. - -## Enriching your Software Catalog - -After services are populated in the catalog, you can enrich your service definitions with additional context to make them more useful. This could include adding key pieces of service metadata to your `service.definition.yaml` files such as: - -- Team -- On-call engineer -- Contact channel -- Documentation links -- Last deployed version -- Code repositories -- Runbook links -- Library dependencies and their versions -- Relevant dashboards - -Software Catalog uses service definition schemas to store and display this metadata about your services. The schemas have built-in validation rules to ensure that only valid values are accepted. There are currently [four supported schemas][11]: v2, v2.1, v2.2, and v3. - -## Using Software Catalog Scorecards - -[Service Scorecards][12] help you codify your organization's best practices as rules that can be evaluated. By implementing scorecards in your catalog, your teams can measure many dimensions of service quality, including: - -- Monitoring coverage -- Production readiness -- Security posture -- Adoption of the latest internal tooling -- Integration checks - -Datadog Scorecards include 10 out-of-the-box rules across observability practices, ownership tagging, and production readiness checkpoints. You can also define your own custom rules. For example, you could create a scorecard that contains a set of rules that map to the steps in your security review process, so that you can quickly audit whether each of your services is compliant. These rules might include checks related to CVE analysis, RBAC configuration, or other security parameters. - -To add custom rules to your Scorecards dashboard: - -1. Click **Create Rule** on the Scorecards page. -2. Specify the name of the rule, the scorecard it belongs to, a rule description, and the owning team. -3. Send an outcome of `pass`, `fail`, or `skip` for each `{rule, service}` tuple that you are evaluating to the [Scorecards API][13] `/scorecard/outcomes/batch` endpoint. -4. View an overview of outcomes in the Scorecards dashboard. - -{{< img src="/getting_started/software_catalog/create_rule.jpeg" alt="Create Rule modal to add custom rules in Scorecards dashboard" style="width:90%;" >}} - -## Further reading - -{{< partial name="whats-next/whats-next.html" >}} - -[1]: https://www.datadoghq.com -[2]: /tracing -[3]: /universal_service_monitoring -[4]: /real_user_monitoring -[5]: /getting_started/tagging/unified_service_tagging -[6]: /integrations/servicenow/#service-ingestion -[7]: /integrations/backstage/ -[8]: https://www.datadoghq.com/blog/github-actions-service-catalog -[9]: https://backstage.io/docs/overview/what-is-backstage -[10]:/api/latest/service-definition -[11]: /software_catalog/add_metadata -[12]: /software_catalog/scorecards -[13]: /api/latest/service-scorecards