[python] Add logic to clear output folder (#8716)

fix Azure/azure-sdk-tools#11791

- For common scenario without `generation-subdir` configured, added test
case in this PR
- For scenario with `generation-subdir` configured, since there is no
`generated` folder, I have to add test case in autorest.python repo
Azure/autorest.python#3239

Author: msyyc

.chronus/changes/clear-output-dir-python-2025-9-11-13-42-52.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/http-client-python"
+---
+
+Add logic to clear output folder

packages/http-client-python/eng/scripts/ci/regenerate.ts

@@ -447,11 +447,35 @@ async function runTaskPool(tasks: Array<() => Promise<void>>, poolLimit: number)
   await Promise.all(workers);
 }
 
+// create some files before regeneration. After regeneration, these files should be deleted and we will test it
+// in test case
+async function preprocess(flags: RegenerateFlagsInput): Promise<void> {
+  if (flags.flavor === "azure") {
+    // create folder if not exists
+    const folderParts = [
+      "test",
+      "azure",
+      "generated",
+      "authentication-api-key",
+      "authentication",
+      "apikey",
+      "_operations",
+    ];
+    await promises.mkdir(join(GENERATED_FOLDER, ...folderParts), { recursive: true });
+    await promises.writeFile(
+      join(GENERATED_FOLDER, ...folderParts, "to_be_deleted.py"),
+      "# This file is to be deleted after regeneration",
+    );
+  }
+}
+
 async function regenerate(flags: RegenerateFlagsInput): Promise<void> {
   if (flags.flavor === undefined) {
     await regenerate({ flavor: "azure", ...flags });
     await regenerate({ flavor: "unbranded", ...flags });
   } else {
+    await preprocess(flags);
+
     const flagsResolved = { debug: false, flavor: flags.flavor, ...flags };
     const subdirectoriesForAzure = await getSubdirectories(AZURE_HTTP_SPECS, flagsResolved);
     const subdirectoriesForNonAzure = await getSubdirectories(HTTP_SPECS, flagsResolved);

packages/http-client-python/generator/pygen/__init__.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import shutil
 from collections.abc import ItemsView, KeysView, MutableMapping, ValuesView
 import logging
 from pathlib import Path
@@ -248,9 +249,23 @@ def remove_file(self, filename: Union[str, Path]) -> None:
         except FileNotFoundError:
             pass
 
+    def remove_folder(self, foldername: Union[str, Path]) -> None:
+        try:
+            folder_path = self.output_folder / Path(foldername)
+            if folder_path.exists() and folder_path.is_dir():
+                shutil.rmtree(folder_path)
+        except FileNotFoundError:
+            pass
+
     def list_file(self) -> list[str]:
         return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
 
+    def list_file_of_folder(self, foldername: Union[str, Path]) -> list[str]:
+        folder_path = self.output_folder / Path(foldername)
+        if folder_path.exists() and folder_path.is_dir():
+            return [str(f.relative_to(self.output_folder)) for f in folder_path.glob("**/*") if f.is_file()]
+        return []
+
 
 class Plugin(ReaderAndWriter, ABC):
     """A base class for autorest plugin.

packages/http-client-python/generator/pygen/codegen/serializers/__init__.py

@@ -122,7 +122,21 @@ def keep_version_file(self) -> bool:
             # If parsing the version fails, we assume the version file is not valid and overwrite.
             return False
 
+    # pylint: disable=too-many-branches
     def serialize(self) -> None:
+        # remove existing folders when generate from tsp
+        if self.code_model.is_tsp and self.code_model.is_azure_flavor:
+            # remove generated_samples and generated_tests folder
+            self.remove_folder(self._generated_tests_samples_folder("generated_samples"))
+            self.remove_folder(self._generated_tests_samples_folder("generated_tests"))
+
+            # remove generated sdk files
+            generation_path = self.code_model.get_generation_dir(self.code_model.namespace)
+            for file in self.list_file_of_folder(generation_path):
+                if file.endswith(".py") and "_patch.py" not in file:
+                    self.remove_file(file)
+
+        # serialize logic
         env = Environment(
             loader=PackageLoader("pygen.codegen", "templates"),
             keep_trailing_newline=True,
@@ -519,8 +533,11 @@ def sample_additional_folder(self) -> Path:
             return Path("/".join(namespace_config.split(".")[num_of_package_namespace:]))
         return Path("")
 
+    def _generated_tests_samples_folder(self, folder_name: str) -> Path:
+        return self._root_of_sdk / folder_name
+
     def _serialize_and_write_sample(self, env: Environment):
-        out_path = self._root_of_sdk / "generated_samples"
+        out_path = self._generated_tests_samples_folder("generated_samples")
         for client in self.code_model.clients:
             for op_group in client.operation_groups:
                 for operation in op_group.operations:
@@ -549,7 +566,7 @@ def _serialize_and_write_sample(self, env: Environment):
 
     def _serialize_and_write_test(self, env: Environment):
         self.code_model.for_test = True
-        out_path = self._root_of_sdk / "generated_tests"
+        out_path = self._generated_tests_samples_folder("generated_tests")
         general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env)
         self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest())
         if not self.code_model.options["azure-arm"]:

packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from pathlib import Path
+
+GENERATED_PATH = Path(__file__).parent.parent.resolve() / "generated"
+
+
+def test_clear_output_folder():
+    folder = GENERATED_PATH / "authentication-api-key/authentication/apikey/_operations"
+    assert folder.exists(), "Operations folder should exist"
+    assert not (folder / "to_be_deleted.py").exists(), "File to_be_deleted.py should be deleted after regeneration"