[Generator] (feat) Added datasource generation

.generator/README.md

@@ -1,9 +1,10 @@
 # Terraform Generation
 
-The goal of this sub-project is to generate the scaffolding to create a Terraform resource.
+The goal of this sub-project is to generate the scaffolding to create a Terraform resource or datasource.
 
 > [!CAUTION]
 > This code is HIGHLY experimental and should stabilize over the next weeks/months. As such this code is NOT intended for production uses.
+> Any code that has been generate should and needs to be proofread by a human.
 
 ## How to use
 
@@ -22,7 +23,7 @@ Install go as we use the `go fmt` command on the generated files to format them.
 
 ### Marking the resources to be generated
 
-The generator reads a configuration file in order to generate the appropriate resources.
+The generator reads a configuration file in order to generate the appropriate resources and datasources.
 The configuration file should look like the following:
 
 ```yaml
@@ -41,14 +42,25 @@ resources:
       method: { delete_method }
       path: { delete_path }
   ...
+
+datasources:
+  { datasource_name }:
+    singular: { get_one_path }
+    plural: { get_all_path }
+  ...
 ```
 
-- `resource_name` is the name of the resource to be generated.
-- `xxx_method` should be the HTTP method used by the relevant route
-- `xxx_path` should be the HTTP route of the resource's CRUD operation
+- Resources
+  - `resource_name` is the name of the resource to be generated.
+  - `xxx_method` should be the HTTP method used by the relevant route
+  - `xxx_path` should be the HTTP route of the resource's CRUD operation
+- Datasources
+  - `datasource_name` is the name of the datasource to be generated.
+  - `get_one_path` should be the api route to get a singular item relevant to the datasource
+  - `get_all_path` should be the api route to get a list of items relevant to the datasource
 
 > [!NOTE]
-> An example using the `team` resource would look like this:
+> An example using the `team` resource and datasource would look like this:
 >
 > ```yaml
 > resources:
@@ -65,6 +77,10 @@ resources:
 >     delete:
 >       method: delete
 >       path: /api/v2/team/{team_id}
+> datasources:
+>   team:
+>     singular: /api/v2/team/{team_id}
+>     plural: /api/v2/team
 > ```
 
 ### Running the generator
@@ -76,4 +92,6 @@ Once the configuration file is written, you can run the following command to gen
 ```
 
 > [!NOTE]
+> The `openapi_spec_path` must be placed in a folder named V1 or V2 depending on the datadog api's version it contains
+>
 > The generated resources will be placed in `datadog/fwprovider/`

.generator/src/generator/cli.py

@@ -34,6 +34,12 @@ def cli(spec_path, config_path, go_fmt):
     spec = setup.load(spec_path)
     config = setup.load(config_path)
 
+    data_sources_to_generate = openapi.get_data_sources(spec, config)
+    for name, data_source in data_sources_to_generate.items():
+        generate_data_source(
+            name=name, data_source=data_source, templates=templates, go_fmt=go_fmt
+        )
+
     resources_to_generate = openapi.get_resources(spec, config)
 
     for name, resource in resources_to_generate.items():
@@ -45,6 +51,17 @@ def cli(spec_path, config_path, go_fmt):
         )
 
 
+def generate_data_source(
+    name: str, data_source: dict, templates: dict[str, Template], go_fmt: bool
+) -> None:
+    output = pathlib.Path("../datadog/")
+    filename = output / f"fwprovider/data_source_datadog_{name}.go"
+    with filename.open("w") as fp:
+        fp.write(templates["datasource"].render(name=name, operations=data_source))
+    if go_fmt:
+        subprocess.call(["go", "fmt", filename])
+
+
 def generate_resource(
     name: str, resource: dict, templates: dict[str, Template], go_fmt: bool
 ) -> None:

.generator/src/generator/formatter.py

@@ -142,7 +142,9 @@ def get_terraform_schema_type(schema):
     }[schema.get("type")]
 
 
-def go_to_terraform_type_formatter(name: str, schema: dict) -> str:
+def go_to_terraform_type_formatter(
+    name: str, schema: dict, pointer: bool = True
+) -> str:
     """
     This function is intended to be used in the Jinja2 templates.
     It was made to support the format enrichment of the OpenAPI schema.
@@ -156,12 +158,16 @@ def go_to_terraform_type_formatter(name: str, schema: dict) -> str:
     """
     match schema.get("format"):
         case "date-time":
-            return f"{variable_name(name)}.String()"
+            return f"{name}.String()"
         case "date":
-            return f"{variable_name(name)}.String()"
+            return f"{name}.String()"
         case "binary":
-            return f"string({variable_name(name)})"
+            return f"string({name})"
+        case "int32":
+            return f"int64({name})"
+        case "int64":
+            return f"int64({name})"
 
         # primitive types should fall through
         case _:
-            return f"*{variable_name(name)}"
+            return f"*{name}" if pointer else f"{name}"

.generator/src/generator/openapi.py

@@ -62,9 +62,26 @@ def get_resources(spec: dict, config: dict) -> dict:
     return resources_to_generate
 
 
-def get_terraform_primary_id(operations):
-    update_params = parameters(operations[UPDATE_OPERATION]["schema"])
-    primary_id = operations[UPDATE_OPERATION]["path"].split("/")[-1][1:-1]
+def get_data_sources(spec: dict, config: dict) -> dict:
+    data_source_to_generate = {}
+    for data_source in config["datasources"]:
+        singular_path = config["datasources"][data_source]["singular"]
+        data_source_to_generate.setdefault(data_source, {})["singular"] = {
+            "schema": spec["paths"][singular_path]["get"],
+            "path": singular_path,
+        }
+        plural_path = config["datasources"][data_source]["plural"]
+        data_source_to_generate.setdefault(data_source, {})["plural"] = {
+            "schema": spec["paths"][plural_path]["get"],
+            "path": plural_path,
+        }
+
+    return data_source_to_generate
+
+
+def get_terraform_primary_id(operations, path=UPDATE_OPERATION):
+    update_params = parameters(operations[path]["schema"])
+    primary_id = operations[path]["path"].split("/")[-1][1:-1]
     primary_id_param = update_params.pop(primary_id)
 
     return {"schema": parameter_schema(primary_id_param), "name": primary_id}

.generator/src/generator/setup.py

@@ -21,15 +21,20 @@ def load_environment(version: str) -> Environment:
     env.filters["snake_case"] = formatter.snake_case
     env.filters["untitle_case"] = formatter.untitle_case
     env.filters["variable_name"] = formatter.variable_name
-    env.filters["date_time_formatter"] = formatter.go_to_terraform_type_formatter
+    env.filters["go_to_terraform_type_formatter"] = (
+        formatter.go_to_terraform_type_formatter
+    )
     env.filters["parameter_schema"] = openapi.parameter_schema
     env.filters["parameters"] = openapi.parameters
     env.filters["is_json_api"] = openapi.is_json_api
     env.filters["capitalize"] = utils.capitalize
     env.filters["is_primitive"] = utils.is_primitive
     env.filters["debug"] = utils.debug_filter
+    env.filters["only_keep_filters"] = utils.only_keep_filters
     env.filters["response_type"] = type.get_type_for_response
+    env.filters["get_schema_from_response"] = type.get_schema_from_response
     env.filters["return_type"] = type.return_type
+    env.filters["sort_schemas_by_type"] = type.sort_schemas_by_type
     env.filters["tf_sort_params_by_type"] = type.tf_sort_params_by_type
     env.filters["tf_sort_properties_by_type"] = type.tf_sort_properties_by_type
 
@@ -62,6 +67,7 @@ def load_templates(env: Environment) -> dict[str, Template]:
         "test": env.get_template("resource_test.j2"),
         "example": env.get_template("resource_example.j2"),
         "import": env.get_template("resource_import_example.j2"),
+        "datasource": env.get_template("data_source/base.j2"),
     }
     return templates
 

.generator/src/generator/templates/data_source/base.j2

@@ -0,0 +1,50 @@
+{%- set apiName = operations["singular"]["schema"]["tags"][0].replace(" ", "") + "Api" %}
+{%- set singularParams = operations["singular"]["schema"]|parameters %}
+{%- set singularParamAttr = singularParams|sort_schemas_by_type%}
+
+{%- set pluralParams = operations["plural"]["schema"]|parameters %}
+{%- set primitiveParamAttr, primitiveParamArrAttr, nonPrimitiveParamListAttr, nonPrimitiveParamObjAttr = pluralParams|only_keep_filters|sort_schemas_by_type %}
+
+{%- set singularResp = operations["singular"]["schema"]["responses"]|get_schema_from_response %}
+{%- set primitiveRespAttr, primitiveRespArrAttr, nonPrimitiveRespListAttr, nonPrimitiveRespObjAttr = singularResp|tf_sort_properties_by_type %}
+
+{%- set primaryId = get_terraform_primary_id(operations, "singular") %}
+
+package fwprovider
+
+import (
+    "context"
+
+	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
+	"github.com/hashicorp/terraform-plugin-framework/datasource"
+	"github.com/hashicorp/terraform-plugin-framework/datasource/schema"
+	"github.com/hashicorp/terraform-plugin-framework/types"
+
+	"github.com/terraform-providers/terraform-provider-datadog/datadog/internal/utils"
+)
+
+var (
+    _ datasource.DataSource = &datadog{{ name|camel_case }}DataSource{}
+)
+
+{% include "data_source/types.j2" %}
+
+func NewDatadog{{ name|camel_case }}DataSource() datasource.DataSource {
+	return &datadog{{ name|camel_case }}DataSource{}
+}
+
+func (d *datadog{{ name|camel_case }}DataSource) Configure(_ context.Context, request datasource.ConfigureRequest, response *datasource.ConfigureResponse) {
+	providerData, _ := request.ProviderData.(*FrameworkProvider)
+	d.Api = providerData.DatadogApiInstances.Get{{ apiName }}{{ version }}()
+	d.Auth = providerData.Auth
+}
+
+func (d *datadog{{ name|camel_case }}DataSource) Metadata(_ context.Context, request datasource.MetadataRequest, response *datasource.MetadataResponse) {
+	response.TypeName = "{{ name|snake_case }}"
+}
+
+{% include "data_source/schema.j2" %}
+
+{% include "data_source/read.j2" %}
+
+{% include "data_source/state.j2" %}

.generator/src/generator/templates/data_source/read.j2

@@ -0,0 +1,48 @@
+func (d *datadog{{ name|camel_case }}DataSource) Read(ctx context.Context, request datasource.ReadRequest, response *datasource.ReadResponse) {
+	var state datadog{{ name|camel_case }}DataSourceModel
+	response.Diagnostics.Append(request.Config.Get(ctx, &state)...)
+	if response.Diagnostics.HasError() {
+		return
+	}
+
+	if !state.{{name|camel_case}}Id.IsNull() {
+		{{name|camel_case|untitle_case}}Id := state.{{name|camel_case}}Id.ValueString()
+		ddResp, _, err := d.Api.Get{{name|camel_case}}(d.Auth, {{name|camel_case|untitle_case}}Id)
+		if err != nil {
+			response.Diagnostics.Append(utils.FrameworkErrorDiag(err, "error getting datadog {{name|camel_case|untitle_case}}"))
+			return
+		}
+
+		d.updateState(ctx, &state, ddResp.Data)
+	} else {
+		{%- for name, schema in primitiveParamAttr.items() %}
+			{{ name|variable_name }} := state.{{ name|camel_case }}.Value{{ get_terraform_schema_type(schema) }}()
+		{%- endfor%}
+
+		optionalParams := datadog{{version}}.List{{name|camel_case}}sOptionalParameters{
+		{%- for name, schema in primitiveParamAttr.items() %}
+			{{name|camel_case}}: &{{name|variable_name}},
+		{%- endfor%}
+		}
+
+		ddResp, _, err := d.Api.List{{name|camel_case}}s(d.Auth, optionalParams)
+		if err != nil {
+			response.Diagnostics.Append(utils.FrameworkErrorDiag(err, "error listing datadog {{name|camel_case|untitle_case}}"))
+			return
+		}
+
+		if len(ddResp.Data) > 1 {
+			response.Diagnostics.AddError("filters returned more than one result, use more specific search criteria", "")
+			return
+		}
+		if len(ddResp.Data) == 0 {
+			response.Diagnostics.AddError("filters returned no results", "")
+			return
+		}
+
+		d.updateStateFromListResponse(ctx, &state, &ddResp.Data[0])
+	} 
+
+    // Save data into Terraform state
+    response.Diagnostics.Append(response.State.Set(ctx, &state)...)
+}

.generator/src/generator/templates/data_source/schema.j2

@@ -0,0 +1,62 @@
+{%- import "utils/schema_helper.j2" as schemaMacros %}
+
+func (d *datadog{{ name|camel_case }}DataSource) Schema(_ context.Context, _ datasource.SchemaRequest, response *datasource.SchemaResponse) {
+	response.Schema = schema.Schema{
+		Description: "Use this data source to retrieve information about an existing Datadog {{ name }}.",
+		Attributes: map[string]schema.Attribute{
+            // Datasource ID
+            "id": utils.ResourceIDAttribute(),
+            // Query Parameters
+            {%- for name, schema in singularParamAttr[0].items() %}
+            {{- schemaMacros.typePrimitiveSchema(name, schema, required=schema.get("required")) }}
+			{%- endfor %}
+
+            {%- if primitiveParamAttr or primitiveParamArrAttr %}
+                {%- for name, schema in primitiveParamAttr.items() %}
+                    {{- schemaMacros.typePrimitiveSchema(name, schema, required=schema.get("required")) }}
+                {%- endfor %}
+
+                {%- for name, schema in primitiveParamArrAttr.items() %}
+                    {{- schemaMacros.typePrimitiveArraySchema(name, schema, required=schema.get("required")) }}
+                {%- endfor %}
+            {%- endif %}
+
+            {%- if primitiveRespAttr or primitiveRespArrAttr %}
+                // Computed values
+                {%- for name, schema in primitiveRespAttr.items() %}
+                    {{- schemaMacros.typePrimitiveSchema(name, schema, required=schema.get("required"), computed=True) }}
+                {%- endfor %}
+
+                {%- for name, schema in primitiveRespArrAttr.items() %}
+                    {{- schemaMacros.typePrimitiveArraySchema(name, schema, required=schema.get("required"), computed=True) }}
+                {%- endfor %}
+            {%- endif %}
+		},
+        {%- if nonPrimitiveParamObjAttr or nonPrimitiveParamListAttr or
+             nonPrimitiveRespObjAttr or nonPrimitiveRespListAttr %}
+		Blocks: map[string]schema.Block{
+            {%- if nonPrimitiveParamObjAttr or nonPrimitiveParamListAttr %}
+                //Query parameters
+                {%- for name, schema in nonPrimitiveParamListAttr.items() %}
+                    {{- schemaMacros.baseBlockListAttrSchemaBuilder(name, schema, required=schema.get("required")) }}
+                {%- endfor %}
+
+                {%- for name, schema in nonPrimitiveParamObjAttr.items() %}
+                    {{- schemaMacros.baseBlockObjAttrSchemaBuilder(name, schema, required=schema.get("required")) }}
+                {%- endfor %}
+            {%- endif %}
+
+            {%- if nonPrimitiveRespObjAttr or nonPrimitiveRespListAttr %}
+                // Computed values 
+                {%- for name, schema in nonPrimitiveRespListAttr.items() %}
+                    {{- schemaMacros.baseBlockListAttrSchemaBuilder(name, schema, required=schema.get("required"), computed=True) }}
+                {%- endfor %}
+
+                {%- for name, schema in nonPrimitiveRespObjAttr.items() %}
+                    {{- schemaMacros.baseBlockObjAttrSchemaBuilder(name, schema, required=schema.get("required"), computed=True) }}
+                {%- endfor %}
+            {%- endif %}
+		},
+        {%- endif %}
+	}
+}