PoC Code - New Bulk API JSON format

[msvinaykumar]

Description

This PR introduces a new BulkAPI format where bulk API metadata is included in the bulk response. This enhancement aims to provide additional context and details within the bulk responses, facilitating better understanding and usability of the API. And also it includes detailed recommendations within the bulk responses, providing more comprehensive and actionable data.

GET request for job_id 3f5c0a4b-ff3a-4d7a-83c3-66aceceb2107:

{
  "summary": {
    "status": "IN_PROGRESS",
    "total_experiments": 27,
    "processed_experiments": 5,
    "notifications": null,
    "input": {
      "filter": null,
      "time_range": null,
      "datasource": "prometheus-1",
      "webhook": null
    },
    "job_id": "65603c30-64ee-4bd2-85db-8328425c3b09",
    "job_start_time": "2025-01-27T11:18:23.865Z",
    "job_end_time": null
  }
}

http://127.0.0.1:8080/bulk?job_id=65603c30-64ee-4bd2-85db-8328425c3b09&include=summary,experiments&experiment_name=prometheus-1|default|cadvisor|cadvisor(daemonset)|cadvisor

{
  "summary": {
    "status": "COMPLETED",
    "total_experiments": 27,
    "processed_experiments": 27,
    "notifications": null,
    "input": {
      "filter": null,
      "time_range": null,
      "datasource": "prometheus-1",
      "webhook": null
    },
    "job_id": "65603c30-64ee-4bd2-85db-8328425c3b09",
    "job_start_time": "2025-01-27T11:18:23.865Z",
    "job_end_time": "2025-01-27T11:18:42.459Z"
  },
  "experiments": {
    "prometheus-1|default|cadvisor|cadvisor(daemonset)|cadvisor": {
      "name": "prometheus-1|default|cadvisor|cadvisor(daemonset)|cadvisor",
      "status": "PROCESSED",
      "apis": {
        "create": {
          "response": {
            "message": "Experiment registered successfully with Kruize. View registered experiments at /listExperiments",
            "httpcode": 201,
            "documentationLink": "",
            "status": "SUCCESS"
          },
          "request": {
            "apiVersion": "v2.0",
            "experimentName": "prometheus-1|default|cadvisor|cadvisor(daemonset)|cadvisor",
            "clusterName": "default",
            "performanceProfile": "resource-optimization-local-monitoring",
            "sloInfo": null,
            "mode": "monitor",
            "targetCluster": "local",
            "trialSettings": {
              "measurement_durationMinutes": "15min",
              "measurement_durationMinutes_inDouble": 15
            },
            "recommendationSettings": {
              "threshold": 0.1
            },
            "datasource": "prometheus-1",
            "experimentType": "CONTAINER",
            "status": "IN_PROGRESS",
            "experiment_id": "a9b2ba3af48b680cfa3dec0d0d7cf38ffb2c86beabf0ed7031c6035a27891ac1",
            "validationData": null,
            "kubernetesObjects": [
              {
                "type": "daemonset",
                "name": "cadvisor",
                "namespace": "cadvisor",
                "containers": [
                  {
                    "container_image_name": "gcr.io/cadvisor/cadvisor:v0.45.0",
                    "container_name": "cadvisor",
                    "metrics": null,
                    "recommendations": null
                  }
                ],
                "namespaces": null
              }
            ],
            "namespaceExperiment": false,
            "containerExperiment": true
          }
        },
        "recommendations": {
          "response": [
            {
              "cluster_name": "default",
              "experiment_type": "container",
              "kubernetes_objects": [
                {
                  "type": "daemonset",
                  "name": "cadvisor",
                  "namespace": "cadvisor",
                  "containers": [
                    {
                      "container_image_name": "gcr.io/cadvisor/cadvisor:v0.45.0",
                      "container_name": "cadvisor",
                      "recommendations": {
                        "version": "1.0",
                        "notifications": {
                          "111000": {
                            "type": "info",
                            "message": "Recommendations Are Available",
                            "code": 111000
                          }
                        },
                        "data": {
                          "2025-01-27T11:28:24.000Z": {
                            "notifications": {
                              "111101": {
                                "type": "info",
                                "message": "Short Term Recommendations Available",
                                "code": 111101
                              },
                              "223001": {
                                "type": "error",
                                "message": "Amount field is missing in the CPU Section",
                                "code": 223001
                              },
                              "224001": {
                                "type": "error",
                                "message": "Amount field is missing in the Memory Section",
                                "code": 224001
                              },
                              "423001": {
                                "type": "warning",
                                "message": "CPU Limit Not Set",
                                "code": 423001
                              },
                              "523001": {
                                "type": "critical",
                                "message": "CPU Request Not Set",
                                "code": 523001
                              },
                              "524001": {
                                "type": "critical",
                                "message": "Memory Request Not Set",
                                "code": 524001
                              },
                              "524002": {
                                "type": "critical",
                                "message": "Memory Limit Not Set",
                                "code": 524002
                              }
                            },
                            "monitoring_end_time": "2025-01-27T11:28:24.000Z",
                            "current": {},
                            "recommendation_terms": {
                              "short_term": {
                                "duration_in_hours": 24,
                                "notifications": {
                                  "112101": {
                                    "type": "info",
                                    "message": "Cost Recommendations Available",
                                    "code": 112101
                                  },
                                  "112102": {
                                    "type": "info",
                                    "message": "Performance Recommendations Available",
                                    "code": 112102
                                  }
                                },
                                "monitoring_start_time": "2025-01-26T11:28:24.000Z",
                                "recommendation_engines": {
                                  "cost": {
                                    "pods_count": 1,
                                    "confidence_level": 0,
                                    "config": {
                                      "limits": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      },
                                      "requests": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      }
                                    },
                                    "variation": {
                                      "limits": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      },
                                      "requests": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      }
                                    },
                                    "notifications": {}
                                  },
                                  "performance": {
                                    "pods_count": 1,
                                    "confidence_level": 0,
                                    "config": {
                                      "limits": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      },
                                      "requests": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      }
                                    },
                                    "variation": {
                                      "limits": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      },
                                      "requests": {
                                        "memory": {
                                          "amount": 136651161.6,
                                          "format": "bytes"
                                        },
                                        "cpu": {
                                          "amount": 0.22053964085707395,
                                          "format": "cores"
                                        }
                                      }
                                    },
                                    "notifications": {}
                                  }
                                },
                                "plots": {
                                  "datapoints": 4,
                                  "plots_data": {
                                    "2025-01-27T11:28:24.000Z": {
                                      "cpuUsage": {
                                        "min": 0.08396913905943296,
                                        "q1": 0.18669564827597168,
                                        "median": 0.19308384827591732,
                                        "q3": 0.2056331399785098,
                                        "max": 0.22053964085707395,
                                        "format": "cores"
                                      },
                                      "memoryUsage": {
                                        "min": 104177664,
                                        "q1": 116555776,
                                        "median": 121192448,
                                        "q3": 122347520,
                                        "max": 123654144,
                                        "format": "bytes"
                                      }
                                    },
                                    "2025-01-26T23:28:24.000Z": {},
                                    "2025-01-26T17:28:24.000Z": {},
                                    "2025-01-27T05:28:24.000Z": {
                                      "cpuUsage": {
                                        "min": 0.09123101016771845,
                                        "q1": 0.1949451069246897,
                                        "median": 0.19694082169152047,
                                        "q3": 0.19694082169152047,
                                        "max": 0.21236305529661514,
                                        "format": "cores"
                                      },
                                      "memoryUsage": {
                                        "min": 111341568,
                                        "q1": 121679872,
                                        "median": 121905152,
                                        "q3": 121905152,
                                        "max": 123506688,
                                        "format": "bytes"
                                      }
                                    }
                                  }
                                }
                              },
                              "medium_term": {
                                "duration_in_hours": 168,
                                "notifications": {
                                  "120001": {
                                    "type": "info",
                                    "message": "There is not enough data available to generate a recommendation.",
                                    "code": 120001
                                  }
                                }
                              },
                              "long_term": {
                                "duration_in_hours": 360,
                                "notifications": {
                                  "120001": {
                                    "type": "info",
                                    "message": "There is not enough data available to generate a recommendation.",
                                    "code": 120001
                                  }
                                }
                              }
                            }
                          }
                        }
                      }
                    }
                  ]
                }
              ],
              "version": "v2.0",
              "experiment_name": "prometheus-1|default|cadvisor|cadvisor(daemonset)|cadvisor"
            }
          ]
        }
      },
      "status_history": [
        {
          "status": "UNPROCESSED",
          "timestamp": "2025-01-27T11:18:24.056Z"
        },
        {
          "status": "PROCESSED",
          "timestamp": "2025-01-27T11:18:27.012Z"
        }
      ]
    }
  }
}

Fixes # (issue)

Type of change

• Bug fix
• New feature
• Docs update
• Breaking change (What changes might users need to make in their application due to this PR?)
• Requires DB changes

How has this been tested?

Please describe the tests that were run to verify your changes and steps to reproduce. Please specify any test configuration required.

• New Test X
• Functional testsuite

Test Configuration

• Kubernetes clusters tested on:

Checklist 🎯

• Followed coding guidelines
• Comments added
• Dependent changes merged
• Documentation updated
• Tests added or updated

Additional information

Include any additional information such as links, test results, screenshots here

[msvinaykumar]

Will Include md file and need to update bulk demo script

[dinogun]

@msvinaykumar I'm assuming this is not a PR for merging but for PoC purposes and for @khansaad to build the kafka functionality on top. Please mark the PR as such

[msvinaykumar]

@msvinaykumar I'm assuming this is not a PR for merging but for PoC purposes and for @khansaad to build the kafka functionality on top. Please mark the PR as such

ok