diff --git a/.doc_gen/metadata/glue_metadata.yaml b/.doc_gen/metadata/glue_metadata.yaml index 9a87b2d6474..b614beff26e 100644 --- a/.doc_gen/metadata/glue_metadata.yaml +++ b/.doc_gen/metadata/glue_metadata.yaml @@ -153,6 +153,15 @@ glue_GetCrawler: - description: snippet_tags: - rust.glue.get_crawler + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.GetCrawler services: glue: {GetCrawler} glue_CreateCrawler: @@ -241,6 +250,15 @@ glue_CreateCrawler: - description: snippet_tags: - rust.glue.create_crawler + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.CreateCrawler services: glue: {CreateCrawler} glue_StartCrawler: @@ -329,6 +347,15 @@ glue_StartCrawler: - description: snippet_tags: - rust.glue.start_crawler + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.StartCrawler services: glue: {StartCrawler} glue_GetDatabase: @@ -416,6 +443,15 @@ glue_GetDatabase: - description: snippet_tags: - rust.glue.get_database + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.GetDatabase services: glue: {GetDatabase} glue_GetTables: @@ -494,6 +530,15 @@ glue_GetTables: - description: snippet_tags: - rust.glue.get_tables + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.GetTables services: glue: {GetTables} glue_CreateJob: @@ -573,6 +618,15 @@ glue_CreateJob: - description: snippet_tags: - rust.glue.create_job + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.CreateJob services: glue: {CreateJob} glue_StartJobRun: @@ -653,6 +707,15 @@ glue_StartJobRun: - description: snippet_tags: - rust.glue.start_job_run + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.StartJobRun services: glue: {StartJobRun} glue_ListJobs: @@ -720,6 +783,15 @@ glue_ListJobs: - description: snippet_tags: - rust.glue.list_jobs + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.ListJobs services: glue: {ListJobs} glue_GetJobRuns: @@ -859,6 +931,15 @@ glue_GetJobRun: - description: snippet_tags: - rust.glue.get_job_run + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.GetJobRun services: glue: {GetJobRun} glue_DeleteJob: @@ -936,6 +1017,15 @@ glue_DeleteJob: - description: snippet_tags: - rust.glue.delete_job + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.DeleteJob services: glue: {DeleteJob} glue_DeleteTable: @@ -1070,6 +1160,15 @@ glue_DeleteDatabase: - description: snippet_tags: - rust.glue.delete_database + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.DeleteDatabase services: glue: {DeleteDatabase} glue_DeleteCrawler: @@ -1147,6 +1246,15 @@ glue_DeleteCrawler: - description: snippet_tags: - rust.glue.delete_crawler + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: + snippet_tags: + - swift.glue.import + - swift.glue.DeleteCrawler services: glue: {DeleteCrawler} glue_GetDatabases: @@ -1333,6 +1441,17 @@ glue_Scenario_GetStartedCrawlersJobs: - rust.glue.delete_table - rust.glue.delete_database - rust.glue.delete_crawler + Swift: + versions: + - sdk_version: 1 + github: swift/example_code/glue + excerpts: + - description: The Package.swift file. + snippet_tags: + - swift.glue.scenario.package + - description: The Swift code file, entry.swift. + snippet_tags: + - swift.glue.scenario services: glue: {GetCrawler, CreateCrawler, StartCrawler, GetDatabase, GetDatabases, GetTables, CreateJob, StartJobRun, ListJobs, GetJob, GetJobRuns, GetJobRun, DeleteJob, DeleteTable, DeleteDatabase, DeleteCrawler} diff --git a/swift/example_code/glue/README.md b/swift/example_code/glue/README.md new file mode 100644 index 00000000000..89f4be3055d --- /dev/null +++ b/swift/example_code/glue/README.md @@ -0,0 +1,123 @@ +# AWS Glue code examples for the SDK for Swift + +## Overview + +Shows how to use the AWS SDK for Swift to work with AWS Glue. + + + + +_AWS Glue is a scalable, serverless data integration service that makes it easy to discover, prepare, and combine data for analytics, machine learning, and application development._ + +## ⚠ Important + +* Running this code might result in charges to your AWS account. For more details, see [AWS Pricing](https://aws.amazon.com/pricing/) and [Free Tier](https://aws.amazon.com/free/). +* Running the tests might result in charges to your AWS account. +* We recommend that you grant your code least privilege. At most, grant only the minimum permissions required to perform the task. For more information, see [Grant least privilege](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#grant-least-privilege). +* This code is not tested in every AWS Region. For more information, see [AWS Regional Services](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services). + + + + +## Code examples + +### Prerequisites + +For prerequisites, see the [README](../../README.md#Prerequisites) in the `swift` folder. + + + + + +### Basics + +Code examples that show you how to perform the essential operations within a service. + +- [Learn the basics](scenario/Package.swift) + + +### Single actions + +Code excerpts that show you how to call individual service functions. + +- [CreateCrawler](scenario/Sources/entry.swift#L134) +- [CreateJob](scenario/Sources/entry.swift#L275) +- [DeleteCrawler](scenario/Sources/entry.swift#L178) +- [DeleteDatabase](scenario/Sources/entry.swift#L463) +- [DeleteJob](scenario/Sources/entry.swift#L349) +- [GetCrawler](scenario/Sources/entry.swift#L220) +- [GetDatabase](scenario/Sources/entry.swift#L399) +- [GetJobRun](scenario/Sources/entry.swift#L557) +- [GetTables](scenario/Sources/entry.swift#L422) +- [ListJobs](scenario/Sources/entry.swift#L312) +- [StartCrawler](scenario/Sources/entry.swift#L198) +- [StartJobRun](scenario/Sources/entry.swift#L518) + + + + + +## Run the examples + +### Instructions + +To build any of these examples from a terminal window, navigate into its +directory, then use the following command: + +``` +$ swift build +``` + +To build one of these examples in Xcode, navigate to the example's directory +(such as the `ListUsers` directory, to build that example). Then type `xed.` +to open the example directory in Xcode. You can then use standard Xcode build +and run commands. + + + + + +#### Learn the basics + +This example shows you how to do the following: + +- Create a crawler that crawls a public Amazon S3 bucket and generates a database of CSV-formatted metadata. +- List information about databases and tables in your AWS Glue Data Catalog. +- Create a job to extract CSV data from the S3 bucket, transform the data, and load JSON-formatted output into another S3 bucket. +- List information about job runs, view transformed data, and clean up resources. + + + + + + + + + +### Tests + +⚠ Running tests might result in charges to your AWS account. + + +To find instructions for running these tests, see the [README](../../README.md#Tests) +in the `swift` folder. + + + + + + +## Additional resources + +- [AWS Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/what-is-glue.html) +- [AWS Glue API Reference](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api.html) +- [SDK for Swift AWS Glue reference](https://sdk.amazonaws.com/swift/api/awsglue/latest/documentation/awsglue) + + + + +--- + +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 diff --git a/swift/example_code/glue/scenario/Package.swift b/swift/example_code/glue/scenario/Package.swift new file mode 100644 index 00000000000..d50b0f74ba9 --- /dev/null +++ b/swift/example_code/glue/scenario/Package.swift @@ -0,0 +1,48 @@ +// swift-tools-version: 5.9 +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// (swift-tools-version has two lines here because it needs to be the first +// line in the file, but it should also appear in the snippet below) +// +// snippet-start:[swift.glue.scenario.package] +// swift-tools-version: 5.9 +// +// The swift-tools-version declares the minimum version of Swift required to +// build this package. + +import PackageDescription + +let package = Package( + name: "glue-scenario", + // Let Xcode know the minimum Apple platforms supported. + platforms: [ + .macOS(.v13), + .iOS(.v15) + ], + dependencies: [ + // Dependencies declare other packages that this package depends on. + .package( + url: "https://github.com/awslabs/aws-sdk-swift", + from: "1.0.0"), + .package( + url: "https://github.com/apple/swift-argument-parser.git", + branch: "main" + ) + ], + targets: [ + // Targets are the basic building blocks of a package, defining a module or a test suite. + // Targets can depend on other targets in this package and products + // from dependencies. + .executableTarget( + name: "glue-scenario", + dependencies: [ + .product(name: "AWSGlue", package: "aws-sdk-swift"), + .product(name: "AWSS3", package: "aws-sdk-swift"), + .product(name: "ArgumentParser", package: "swift-argument-parser") + ], + path: "Sources") + + ] +) +// snippet-end:[swift.glue.scenario.package] diff --git a/swift/example_code/glue/scenario/Sources/entry.swift b/swift/example_code/glue/scenario/Sources/entry.swift new file mode 100644 index 00000000000..f2b9418c2d6 --- /dev/null +++ b/swift/example_code/glue/scenario/Sources/entry.swift @@ -0,0 +1,850 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// snippet-start:[swift.glue.scenario] +// An example that shows how to use the AWS SDK for Swift to demonstrate +// creating and using crawlers and jobs using AWS Glue. +// +// 0. Upload the Python job script to Amazon S3 so it can be used when +// calling `startJobRun()` later. +// 1. Create a crawler, pass it the IAM role and the URL of the public Amazon +// S3 bucket that contains the source data: +// s3://crawler-public-us-east-1/flight/2016/csv. +// 2. Start the crawler. This takes time, so after starting it, use a loop +// that calls `getCrawler()` until the state is "READY". +// 3. Get the database created by the crawler, and the tables in the +// database. Display them to the user. +// 4. Create a job. Pass it the IAM role and the URL to a Python ETL script +// previously uploaded to the user's S3 bucket. +// 5. Start a job run, passing the following custom arguments. These are +// expected by the ETL script, so must exactly match. +// * `--input_database: ` +// * `--input_table: ` +// * `--output_bucket_url: ` +// 6. Loop and get the job run until it returns one of the following states: +// "SUCCEEDED", "STOPPED", "FAILED", or "TIMEOUT". +// 7. Output data is stored in a group of files in the user's S3 bucket. +// Either direct the user to their location or download a file and display +// the results inline. +// 8. List the jobs for the user's account. +// 9. Get job run details for a job run. +// 10. Delete the demo job. +// 11. Delete the database and tables created by the example. +// 12. Delete the crawler created by the example. + +import ArgumentParser +import AWSS3 +import Foundation +import Smithy + +// snippet-start:[swift.glue.import] +import AWSClientRuntime +import AWSGlue +// snippet-end:[swift.glue.import] + +struct ExampleCommand: ParsableCommand { + @Option(help: "The AWS IAM role to use for AWS Glue calls.") + var role: String + + @Option(help: "The Amazon S3 bucket to use for this example.") + var bucket: String + + @Option(help: "The Amazon S3 URL of the data to crawl.") + var s3url: String = "s3://crawler-public-us-east-1/flight/2016/csv" + + @Option(help: "The Python script to run as a job with AWS Glue.") + var script: String = "./flight_etl_job_script.py" + + @Option(help: "The AWS Region to run AWS API calls in.") + var awsRegion = "us-east-1" + + @Option(help: "A prefix string to use when naming tables.") + var tablePrefix = "swift-glue-basics-table" + + @Option( + help: ArgumentHelp("The level of logging for the Swift SDK to perform."), + completion: .list([ + "critical", + "debug", + "error", + "info", + "notice", + "trace", + "warning" + ]) + ) + var logLevel: String = "error" + + static var configuration = CommandConfiguration( + commandName: "glue-scenario", + abstract: """ + Demonstrates various features of AWS Glue. + """, + discussion: """ + An example showing how to use AWS Glue to create, run, and monitor + crawlers and jobs. + """ + ) + + /// Generate and return a unique file name that begins with the specified + /// string. + /// + /// - Parameters: + /// - prefix: Text to use at the beginning of the returned name. + /// + /// - Returns: A string containing a unique filename that begins with the + /// specified `prefix`. + /// + /// The returned name uses a random number between 1 million and 1 billion to + /// provide reasonable certainty of uniqueness for the purposes of this + /// example. + func tempName(prefix: String) -> String { + return "\(prefix)-\(Int.random(in: 1000000..<1000000000))" + } + + /// Upload a file to an Amazon S3 bucket. + /// + /// - Parameters: + /// - s3Client: The S3 client to use when uploading the file. + /// - path: The local path of the source file to upload. + /// - toBucket: The name of the S3 bucket into which to upload the file. + /// - key: The key (name) to give the file in the S3 bucket. + /// + /// - Returns: `true` if the file is uploaded successfully, otherwise `false`. + func uploadFile(s3Client: S3Client, path: String, toBucket: String, key: String) async -> Bool { + do { + let fileData: Data = try Data(contentsOf: URL(fileURLWithPath: path)) + let dataStream = ByteStream.data(fileData) + _ = try await s3Client.putObject( + input: PutObjectInput( + body: dataStream, + bucket: toBucket, + key: key + ) + ) + } catch { + print("*** An unexpected error occurred uploading the script to the Amazon S3 bucket \"\(bucket)\".") + return false + } + + return true + } + + // snippet-start:[swift.glue.CreateCrawler] + /// Create a new AWS Glue crawler. + /// + /// - Parameters: + /// - glueClient: An AWS Glue client to use for the crawler. + /// - crawlerName: A name for the new crawler. + /// - iamRole: The name of an Amazon IAM role for the crawler to use. + /// - s3Path: The path of an Amazon S3 folder to use as a target location. + /// - cronSchedule: A `cron` schedule indicating when to run the crawler. + /// - databaseName: The name of an AWS Glue database to operate on. + /// + /// - Returns: `true` if the crawler is created successfully, otherwise `false`. + func createCrawler(glueClient: GlueClient, crawlerName: String, iamRole: String, + s3Path: String, cronSchedule: String, databaseName: String) async -> Bool { + let s3Target = GlueClientTypes.S3Target(path: s3url) + let targetList = GlueClientTypes.CrawlerTargets(s3Targets: [s3Target]) + + do { + _ = try await glueClient.createCrawler( + input: CreateCrawlerInput( + databaseName: databaseName, + description: "Created by the AWS SDK for Swift Scenario Example for AWS Glue.", + name: crawlerName, + role: iamRole, + schedule: cronSchedule, + tablePrefix: tablePrefix, + targets: targetList + ) + ) + } catch _ as AlreadyExistsException { + print("*** A crawler named \"\(crawlerName)\" already exists.") + return false + } catch _ as OperationTimeoutException { + print("*** The attempt to create the AWS Glue crawler timed out.") + return false + } catch { + print("*** An unexpected error occurred creating the AWS Glue crawler: \(error.localizedDescription)") + return false + } + + return true + } + // snippet-end:[swift.glue.CreateCrawler] + + // snippet-start:[swift.glue.DeleteCrawler] + /// Delete an AWS Glue crawler. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - name: The name of the crawler to delete. + /// + /// - Returns: `true` if successful, otherwise `false`. + func deleteCrawler(glueClient: GlueClient, name: String) async -> Bool { + do { + _ = try await glueClient.deleteCrawler( + input: DeleteCrawlerInput(name: name) + ) + } catch { + return false + } + return true + } + // snippet-end:[swift.glue.DeleteCrawler] + + // snippet-start:[swift.glue.StartCrawler] + /// Start running an AWS Glue crawler. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use when starting the crawler. + /// - name: The name of the crawler to start running. + /// + /// - Returns: `true` if the crawler is started successfully, otherwise `false`. + func startCrawler(glueClient: GlueClient, name: String) async -> Bool { + do { + _ = try await glueClient.startCrawler( + input: StartCrawlerInput(name: name) + ) + } catch { + print("*** An unexpected error occurred starting the crawler.") + return false + } + + return true + } + // snippet-end:[swift.glue.StartCrawler] + + // snippet-start:[swift.glue.GetCrawler] + /// Get the state of the specified AWS Glue crawler. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - name: The name of the crawler whose state should be returned. + /// + /// - Returns: A `GlueClientTypes.CrawlerState` value describing the + /// state of the crawler. + func getCrawlerState(glueClient: GlueClient, name: String) async -> GlueClientTypes.CrawlerState { + do { + let output = try await glueClient.getCrawler( + input: GetCrawlerInput(name: name) + ) + + // If the crawler or its state is `nil`, report that the crawler + // is stopping. This may not be what you want for your + // application but it works for this one! + + guard let crawler = output.crawler else { + return GlueClientTypes.CrawlerState.stopping + } + guard let state = crawler.state else { + return GlueClientTypes.CrawlerState.stopping + } + return state + } catch { + return GlueClientTypes.CrawlerState.stopping + } + } + // snippet-end:[swift.glue.GetCrawler] + + // snippet-start:[swift.glue.getCrawlerState] + /// Wait until the specified crawler is ready to run. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - name: The name of the crawler to wait for. + /// + /// - Returns: `true` if the crawler is ready, `false` if the client is + /// stopping (and will therefore never be ready). + func waitUntilCrawlerReady(glueClient: GlueClient, name: String) async -> Bool { + while true { + let state = await getCrawlerState(glueClient: glueClient, name: name) + + if state == .ready { + return true + } else if state == .stopping { + return false + } + Thread.sleep(forTimeInterval: 4) + } + } + // snippet-end:[swift.glue.getCrawlerState] + + // snippet-start:[swift.glue.CreateJob] + /// Create a new AWS Glue job. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - jobName: The name to give the new job. + /// - role: The IAM role for the job to use when accessing AWS services. + /// - scriptLocation: The AWS S3 URI of the script to be run by the job. + /// + /// - Returns: `true` if the job is created successfully, otherwise `false`. + func createJob(glueClient: GlueClient, name jobName: String, role: String, + scriptLocation: String) async -> Bool { + let command = GlueClientTypes.JobCommand( + name: "glueetl", + pythonVersion: "3", + scriptLocation: scriptLocation + ) + + do { + _ = try await glueClient.createJob( + input: CreateJobInput( + command: command, + description: "Created by the AWS SDK for Swift Glue basic scenario example.", + glueVersion: "3.0", + name: jobName, + numberOfWorkers: 10, + role: role, + workerType: .g1x + ) + ) + } catch { + return false + } + return true + } + // snippet-end:[swift.glue.CreateJob] + + // snippet-start:[swift.glue.ListJobs] + /// Return a list of the AWS Glue jobs listed on the user's account. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - maxJobs: The maximum number of jobs to return (default: 100). + /// + /// - Returns: An array of strings listing the names of all available AWS + /// Glue jobs. + func listJobs(glueClient: GlueClient, maxJobs: Int = 100) async -> [String] { + var jobList: [String] = [] + var nextToken: String? + + repeat { + do { + let output = try await glueClient.listJobs( + input: ListJobsInput( + maxResults: maxJobs, + nextToken: nextToken + ) + ) + + guard let jobs = output.jobNames else { + return jobList + } + + jobList = jobList + jobs + nextToken = output.nextToken + } catch { + return jobList + } + } while (nextToken != nil) + + return jobList + } + // snippet-end:[swift.glue.ListJobs] + + // snippet-start:[swift.glue.DeleteJob] + /// Delete an AWS Glue job. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - jobName: The name of the job to delete. + /// + /// - Returns: `true` if the job is successfully deleted, otherwise `false`. + func deleteJob(glueClient: GlueClient, name jobName: String) async -> Bool { + do { + _ = try await glueClient.deleteJob( + input: DeleteJobInput(jobName: jobName) + ) + } catch { + return false + } + return true + } + // snippet-end:[swift.glue.DeleteJob] + + // snippet-start:[swift.glue.CreateDatabase] + /// Create an AWS Glue database. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - databaseName: The name to give the new database. + /// - location: The URL of the source data to use with AWS Glue. + /// + /// - Returns: `true` if the database is created successfully, otherwise `false`. + func createDatabase(glueClient: GlueClient, name databaseName: String, location: String) async -> Bool { + let databaseInput = GlueClientTypes.DatabaseInput( + description: "Created by the AWS SDK for Swift Glue basic scenario example.", + locationUri: location, + name: databaseName + ) + + do { + _ = try await glueClient.createDatabase( + input: CreateDatabaseInput( + databaseInput: databaseInput + ) + ) + } catch { + return false + } + + return true + } + // snippet-end:[swift.glue.CreateDatabase] + + // snippet-start:[swift.glue.GetDatabase] + /// Get the AWS Glue database with the specified name. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - name: The name of the database to return. + /// + /// - Returns: The `GlueClientTypes.Database` object describing the + /// specified database, or `nil` if an error occurs or the database + /// isn't found. + func getDatabase(glueClient: GlueClient, name: String) async -> GlueClientTypes.Database? { + do { + let output = try await glueClient.getDatabase( + input: GetDatabaseInput(name: name) + ) + + return output.database + } catch { + return nil + } + } + // snippet-end:[swift.glue.GetDatabase] + + // snippet-start:[swift.glue.GetTables] + /// Returns a list of the tables in the specified database. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - databaseName: The name of the database whose tables are to be + /// returned. + /// + /// - Returns: An array of `GlueClientTypes.Table` objects, each + /// describing one table in the named database. An empty array indicates + /// that there are either no tables in the database, or an error + /// occurred before any tables could be found. + func getTablesInDatabase(glueClient: GlueClient, databaseName: String) async -> [GlueClientTypes.Table] { + var tables: [GlueClientTypes.Table] = [] + var nextToken: String? + + repeat { + do { + let output = try await glueClient.getTables( + input: GetTablesInput( + databaseName: databaseName, + nextToken: nextToken + ) + ) + + guard let tableList = output.tableList else { + return tables + } + + tables = tables + tableList + nextToken = output.nextToken + } catch { + return tables + } + } while nextToken != nil + + return tables + } + // snippet-end:[swift.glue.GetTables] + + // snippet-start:[swift.glue.BatchDeleteTable] + // snippet-start:[swift.glue.DeleteDatabase] + /// Delete the specified database. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - databaseName: The name of the database to delete. + /// - deleteTables: A Bool indicating whether or not to delete the + /// tables in the database before attempting to delete the database. + /// + /// - Returns: `true` if the database (and optionally its tables) are + /// deleted, otherwise `false`. + func deleteDatabase(glueClient: GlueClient, name databaseName: String, + withTables deleteTables: Bool = false) async -> Bool { + if deleteTables { + var tableNames: [String] = [] + + // Get a list of the names of all of the tables in the database. + + let tableList = await self.getTablesInDatabase(glueClient: glueClient, databaseName: databaseName) + for table in tableList { + guard let name = table.name else { + continue + } + tableNames.append(name) + } + + // Delete the tables. + + do { + _ = try await glueClient.batchDeleteTable( + input: BatchDeleteTableInput( + databaseName: databaseName, + tablesToDelete: tableNames + ) + ) + } catch { + print("*** Unable to delete the tables.") + } + return true + } + + // Delete the database itself. + + do { + _ = try await glueClient.deleteDatabase( + input: DeleteDatabaseInput(name: databaseName) + ) + } catch { + print("*** Unable to delete the database.") + } + return true + } + // snippet-end:[swift.glue.DeleteDatabase] + // snippet-end:[swift.glue.BatchDeleteTable] + + // snippet-start:[swift.glue.StartJobRun] + /// Start an AWS Glue job run. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - jobName: The name of the job to run. + /// - databaseName: The name of the AWS Glue database to run the job against. + /// - tableName: The name of the table in the database to run the job against. + /// - outputURL: The AWS S3 URI of the bucket location into which to + /// write the resulting output. + /// + /// - Returns: `true` if the job run is started successfully, otherwise `false`. + func startJobRun(glueClient: GlueClient, name jobName: String, databaseName: String, + tableName: String, outputURL: String) async -> String? { + do { + let output = try await glueClient.startJobRun( + input: StartJobRunInput( + arguments: [ + "--input_database": databaseName, + "--input_table": tableName, + "--output_bucket_url": outputURL + ], + jobName: jobName, + numberOfWorkers: 10, + workerType: .g1x + ) + ) + + guard let id = output.jobRunId else { + return nil + } + + return id + } catch { + return nil + } + } + // snippet-end:[swift.glue.StartJobRun] + + // snippet-start:[swift.glue.GetJobRun] + /// Get information about a specific AWS Glue job run. + /// + /// - Parameters: + /// - glueClient: The AWS Glue client to use. + /// - jobName: The name of the job to return job run data for. + /// - id: The run ID of the specific job run to return. + /// + /// - Returns: A `GlueClientTypes.JobRun` object describing the state of + /// the job run, or `nil` if an error occurs. + func getJobRun(glueClient: GlueClient, name jobName: String, id: String) async -> GlueClientTypes.JobRun? { + do { + let output = try await glueClient.getJobRun( + input: GetJobRunInput( + jobName: jobName, + runId: id + ) + ) + + return output.jobRun + } catch { + return nil + } + } + // snippet-end:[swift.glue.GetJobRun] + + /// Called by ``main()`` to run the bulk of the example. + func runAsync() async throws { + // A name to give the Python script upon upload to the Amazon S3 + // bucket. + let scriptName = "jobscript.py" + + // Schedule string in `cron` format, as described here: + // https://docs.aws.amazon.com/glue/latest/dg/monitor-data-warehouse-schedule.html + let cron = "cron(15 12 * * ? *)" + + let glueConfig = try await GlueClient.GlueClientConfiguration(region: awsRegion) + let glueClient = GlueClient(config: glueConfig) + + let s3Config = try await S3Client.S3ClientConfiguration(region: awsRegion) + let s3Client = S3Client(config: s3Config) + + // Create random names for things that need them. + + let crawlerName = tempName(prefix: "swift-glue-basics-crawler") + let databaseName = tempName(prefix: "swift-glue-basics-db") + + // Create a name for the AWS Glue job. + + let jobName = tempName(prefix: "scenario-job") + + // The URL of the Python script on S3. + + let scriptURL = "s3://\(bucket)/\(scriptName)" + + print("Welcome to the AWS SDK for Swift basic scenario for AWS Glue!") + + //===================================================================== + // 0. Upload the Python script to the target bucket so it's available + // for use by the Amazon Glue service. + //===================================================================== + + print("Uploading the Python script: \(script) as key \(scriptName)") + print("Destination bucket: \(bucket)") + if !(await uploadFile(s3Client: s3Client, path: script, toBucket: bucket, key: scriptName)) { + return + } + + //===================================================================== + // 1. Create the database and crawler using the randomized names + // generated previously. + //===================================================================== + + print("Creating database \"\(databaseName)\"...") + if !(await createDatabase(glueClient: glueClient, name: databaseName, location: s3url)) { + print("*** Unable to create the database.") + return + } + + print("Creating crawler \"\(crawlerName)\"...") + if !(await createCrawler(glueClient: glueClient, crawlerName: crawlerName, + iamRole: role, s3Path: s3url, cronSchedule: cron, + databaseName: databaseName)) { + return + } + + //===================================================================== + // 2. Start the crawler, then wait for it to be ready. + //===================================================================== + + print("Starting the crawler and waiting until it's ready...") + if !(await startCrawler(glueClient: glueClient, name: crawlerName)) { + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + if !(await waitUntilCrawlerReady(glueClient: glueClient, name: crawlerName)) { + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + } + + //===================================================================== + // 3. Get the database and table created by the crawler. + //===================================================================== + + print("Getting the crawler's database...") + let database = await getDatabase(glueClient: glueClient, name: databaseName) + let tableList = await getTablesInDatabase(glueClient: glueClient, databaseName: databaseName) + + print("Found \(tableList.count) table(s):") + for table in tableList { + print(" \(table.name ?? "")") + } + + if tableList.count != 1 { + print("*** Incorrect number of tables found. There should only be one.") + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + guard let tableName = tableList[0].name else { + print("*** Table is unnamed.") + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + //===================================================================== + // 4. Create a job. + //===================================================================== + + print("Creating a job...") + if !(await createJob(glueClient: glueClient, name: jobName, role: role, + scriptLocation: scriptURL)) { + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + //===================================================================== + // 5. Start a job run. + //===================================================================== + + print("Starting the job...") + + // Construct the Amazon S3 URL for the job run's output. This is in + // the bucket specified on the command line, with a folder name that's + // unique for this job run. + + let timeStamp = Date().timeIntervalSince1970 + let jobPath = "\(jobName)-\(Int(timeStamp))" + let outputURL = "s3://\(bucket)/\(jobPath)" + + // Start the job run. + + let jobRunID = await startJobRun(glueClient: glueClient, name: jobName, + databaseName: databaseName, + tableName: tableName, + outputURL: outputURL) + + guard let jobRunID else { + print("*** Job run ID is invalid.") + _ = await deleteJob(glueClient: glueClient, name: jobName) + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + //===================================================================== + // 6. Wait for the job run to indicate that the run is complete. + //===================================================================== + + print("Waiting for job run to end...") + + var jobRunFinished = false + var jobRunState: GlueClientTypes.JobRunState + + repeat { + let jobRun = await getJobRun(glueClient: glueClient, name: jobName, id: jobRunID) + guard let jobRun else { + print("*** Unable to get the job run.") + _ = await deleteJob(glueClient: glueClient, name: jobName) + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + jobRunState = jobRun.jobRunState ?? .failed + + //===================================================================== + // 7. Output where to find the data if the job run was successful. + // If the job run failed for any reason, output an appropriate + // error message. + //===================================================================== + + switch jobRunState { + case .succeeded: + print("Job run succeeded. JSON files are in the Amazon S3 path:") + print(" \(outputURL)") + jobRunFinished = true + case .stopped: + jobRunFinished = true + case .error: + print("*** Error: Job run ended in an error. \(jobRun.errorMessage ?? "")") + jobRunFinished = true + case .failed: + print("*** Error: Job run failed. \(jobRun.errorMessage ?? "")") + jobRunFinished = true + case .timeout: + print("*** Warning: Job run timed out.") + jobRunFinished = true + default: + Thread.sleep(forTimeInterval: 0.25) + } + } while jobRunFinished != true + + //===================================================================== + // 8. List the jobs for the user's account. + //===================================================================== + + print("\nThe account has the following jobs:") + let jobs = await listJobs(glueClient: glueClient) + + if jobs.count == 0 { + print(" ") + } else { + for job in jobs { + print(" \(job)") + } + } + + //===================================================================== + // 9. Get the job run details for a job run. + //===================================================================== + + print("Information about the job run:") + let jobRun = await getJobRun(glueClient: glueClient, name: jobName, id: jobRunID) + + guard let jobRun else { + print("*** Unable to retrieve the job run.") + _ = await deleteJob(glueClient: glueClient, name: jobName) + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + _ = await deleteCrawler(glueClient: glueClient, name: crawlerName) + return + } + + let startDate = jobRun.startedOn ?? Date(timeIntervalSince1970: 0) + let endDate = jobRun.completedOn ?? Date(timeIntervalSince1970: 0) + let dateFormatter: DateFormatter = DateFormatter() + dateFormatter.dateStyle = .long + dateFormatter.timeStyle = .long + + print(" Started at: \(dateFormatter.string(from: startDate))") + print(" Completed at: \(dateFormatter.string(from: endDate))") + + //===================================================================== + // 10. Delete the job. + //===================================================================== + + print("\nDeleting the job...") + _ = await deleteJob(glueClient: glueClient, name: jobName) + + //===================================================================== + // 11. Delete the database and tables created by this example. + //===================================================================== + + print("Deleting the database...") + _ = await deleteDatabase(glueClient: glueClient, name: databaseName, withTables: true) + + //===================================================================== + // 12. Delete the crawler. + //===================================================================== + + print("Deleting the crawler...") + if !(await deleteCrawler(glueClient: glueClient, name: crawlerName)) { + return + } + } +} + +/// The program's asynchronous entry point. +@main +struct Main { + static func main() async { + let args = Array(CommandLine.arguments.dropFirst()) + + do { + let command = try ExampleCommand.parse(args) + try await command.runAsync() + } catch { + ExampleCommand.exit(withError: error) + } + } +} +// snippet-end:[swift.glue.scenario] diff --git a/swift/example_code/glue/scenario/flight_etl_job_script.py b/swift/example_code/glue/scenario/flight_etl_job_script.py new file mode 100644 index 00000000000..624fc0204c8 --- /dev/null +++ b/swift/example_code/glue/scenario/flight_etl_job_script.py @@ -0,0 +1,81 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +This script is used by the AWS Glue _getting started with crawlers and jobs_ +scenario to perform extract, transform, and load (ETL) operations on sample +flight data. As part of the example, it is uploaded to an Amazon Simple +Storage Service (Amazon S3) bucket so that AWS Glue can access it. +""" + +# pylint: disable=undefined-variable + +# snippet-start:[glue.swift.basics.job-script] +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job + +""" +These custom arguments must be passed as Arguments to the StartJobRun request. + --input_database The name of a metadata database that is contained in + your AWS Glue Data Catalog and that contains tables + that describe the data to be processed. + --input_table The name of a table in the database that describes the + data to be processed. + --output_bucket_url An S3 bucket that receives the transformed output + data. +""" +args = getResolvedOptions(sys.argv, [ + "JOB_NAME", "input_database", "input_table", "output_bucket_url"]) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) + +# Script generated for node S3 Flight Data. +S3FlightData_node1 = glueContext.create_dynamic_frame.from_catalog( + database=args['input_database'], + table_name=args['input_table'], + transformation_ctx="S3FlightData_node1", +) + +# This mapping performs two main functions: +# 1. It simplifies the output by removing most of the fields from the data. +# 2. It renames some fields. For example, `fl_date` is renamed to `flight_date`. +ApplyMapping_node2 = ApplyMapping.apply( + frame=S3FlightData_node1, + mappings=[ + ("year", "long", "year", "long"), + ("month", "long", "month", "tinyint"), + ("day_of_month", "long", "day", "tinyint"), + ("fl_date", "string", "flight_date", "string"), + ("carrier", "string", "carrier", "string"), + ("fl_num", "long", "flight_num", "long"), + ("origin_city_name", "string", "origin_city_name", "string"), + ("origin_state_abr", "string", "origin_state_abr", "string"), + ("dest_city_name", "string", "dest_city_name", "string"), + ("dest_state_abr", "string", "dest_state_abr", "string"), + ("dep_time", "long", "departure_time", "long"), + ("wheels_off", "long", "wheels_off", "long"), + ("wheels_on", "long", "wheels_on", "long"), + ("arr_time", "long", "arrival_time", "long"), + ("mon", "string", "mon", "string"), + ], + transformation_ctx="ApplyMapping_node2", +) + +# Script generated for node Revised Flight Data. +RevisedFlightData_node3 = glueContext.write_dynamic_frame.from_options( + frame=ApplyMapping_node2, + connection_type="s3", + format="json", + connection_options={"path": args['output_bucket_url'], "partitionKeys": []}, + transformation_ctx="RevisedFlightData_node3", +) + +job.commit() +# snippet-end:[glue.swift.basics.job-script] diff --git a/swift/example_code/glue/scenario/scaffold.py b/swift/example_code/glue/scenario/scaffold.py new file mode 100644 index 00000000000..67e598a22a6 --- /dev/null +++ b/swift/example_code/glue/scenario/scaffold.py @@ -0,0 +1,124 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Deploys and destroys scaffold resources by using an AWS CloudFormation stack. +""" + +import argparse + +import boto3 +from botocore.exceptions import ClientError + + +def deploy(cloud_formation_script, stack_name, cf_resource): + """ + Deploys scaffold resources used by the example. The resources are + defined in the CloudFormation script. They're deployed as a CloudFormation stack + so you can manage and destroy them by using CloudFormation actions. + + :param cloud_formation_script: The path to a CloudFormation script. + :param stack_name: The name of the CloudFormation stack. + :param cf_resource: A Boto3 CloudFormation resource. + :return: A dict of outputs from the stack. + """ + with open(cloud_formation_script) as setup_file: + setup_template = setup_file.read() + print(f"Creating {stack_name}.") + stack = cf_resource.create_stack( + StackName=stack_name, + TemplateBody=setup_template, + Capabilities=["CAPABILITY_NAMED_IAM"], + ) + print("Waiting for stack to deploy. This typically takes a minute or two.") + waiter = cf_resource.meta.client.get_waiter("stack_create_complete") + waiter.wait(StackName=stack.name) + stack.load() + print(f"Stack status: {stack.stack_status}") + print("Created resources:") + for resource in stack.resource_summaries.all(): + print(f"\t{resource.resource_type}, {resource.physical_resource_id}") + print("Outputs:") + outputs = {} + for oput in stack.outputs: + outputs[oput["OutputKey"]] = oput["OutputValue"] + print(f"\t{oput['OutputKey']}: {oput['OutputValue']}") + return outputs + + +def destroy(stack, cf_resource, s3_resource): + """ + Destroys the resources managed by the CloudFormation stack, and the CloudFormation + stack itself. + + :param stack: The CloudFormation stack that manages the example resources. + :param cf_resource: A Boto3 CloudFormation resource. + :param s3_resource: A Boto3 S3 resource. + """ + bucket_name = None + for oput in stack.outputs: + if oput["OutputKey"] == "BucketName": + bucket_name = oput["OutputValue"] + if bucket_name is not None: + print(f"Deleting all objects in bucket {bucket_name}.") + s3_resource.Bucket(bucket_name).objects.delete() + print(f"Deleting {stack.name}.") + stack.delete() + print("Waiting for stack removal.") + waiter = cf_resource.meta.client.get_waiter("stack_delete_complete") + waiter.wait(StackName=stack.name) + print("Stack delete complete.") + + +def main(): + parser = argparse.ArgumentParser( + description="Deploys and destroys scaffold resources for the 'Getting started " + "with crawlers and jobs' scenario. Run with the 'deploy' action to " + "deploy resources or with the 'destroy' action to destroy resources." + ) + parser.add_argument( + "action", + choices=["deploy", "destroy"], + help="Indicates the action that the script performs.", + ) + parser.add_argument( + "--script", + default="setup_scenario_getting_started.yaml", + help="The name of the CloudFormation script to use to deploy resources.", + ) + args = parser.parse_args() + + print("-" * 88) + print("Welcome to the AWS Glue getting started with crawlers and jobs scenario.") + print("-" * 88) + + cf_resource = boto3.resource("cloudformation") + stack = cf_resource.Stack("doc-example-glue-scenario-stack") + + try: + if args.action == "deploy": + print("Deploying scaffold resources for the example.") + outputs = deploy(args.script, stack.name, cf_resource) + print("-" * 88) + print("To run the scenario, pass the role and bucket names to it: ") + print( + f"\tpython scenario_getting_started_crawlers_and_jobs.py " + f"{outputs['RoleName']} {outputs['BucketName']}" + ) + print("-" * 88) + print( + "To clean up all AWS resources created for the example, run this script " + "again with the 'destroy' flag." + ) + elif args.action == "destroy": + print("Destroying scaffold resources created for the example.") + destroy(stack, cf_resource, boto3.resource("s3")) + except ClientError as err: + print(f"Something went wrong while trying to {args.action} the stack:") + print(f"{err.response['Error']['Code']}: {err.response['Error']['Message']}") + + print("-" * 88) + + +if __name__ == "__main__": + main() diff --git a/swift/example_code/glue/scenario/setup_scenario_getting_started.yaml b/swift/example_code/glue/scenario/setup_scenario_getting_started.yaml new file mode 100644 index 00000000000..a9ffafdf67b --- /dev/null +++ b/swift/example_code/glue/scenario/setup_scenario_getting_started.yaml @@ -0,0 +1,164 @@ +Resources: + AWSGlueServiceRoleDocExampleF579DC66: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: glue.amazonaws.com + Version: "2012-10-17" + ManagedPolicyArns: + - Fn::Join: + - "" + - - "arn:" + - Ref: AWS::Partition + - :iam::aws:policy/service-role/AWSGlueServiceRole + RoleName: AWSGlueServiceRole-DocExample + Metadata: + aws:cdk:path: doc-example-glue-scenario-stack/AWSGlueServiceRole-DocExample/Resource + AWSGlueServiceRoleDocExampleDefaultPolicy7ECE7D11: + Type: AWS::IAM::Policy + Properties: + PolicyDocument: + Statement: + - Action: + - s3:GetObject* + - s3:GetBucket* + - s3:List* + - s3:DeleteObject* + - s3:PutObject + - s3:PutObjectLegalHold + - s3:PutObjectRetention + - s3:PutObjectTagging + - s3:PutObjectVersionTagging + - s3:Abort* + Effect: Allow + Resource: + - Fn::GetAtt: + - docexampleglue6E2F12E5 + - Arn + - Fn::Join: + - "" + - - Fn::GetAtt: + - docexampleglue6E2F12E5 + - Arn + - /* + Version: "2012-10-17" + PolicyName: AWSGlueServiceRoleDocExampleDefaultPolicy7ECE7D11 + Roles: + - Ref: AWSGlueServiceRoleDocExampleF579DC66 + Metadata: + aws:cdk:path: doc-example-glue-scenario-stack/AWSGlueServiceRole-DocExample/DefaultPolicy/Resource + docexampleglue6E2F12E5: + Type: AWS::S3::Bucket + UpdateReplacePolicy: Delete + DeletionPolicy: Delete + Metadata: + aws:cdk:path: doc-example-glue-scenario-stack/doc-example-glue/Resource + CDKMetadata: + Type: AWS::CDK::Metadata + Properties: + Analytics: v2:deflate64:H4sIAAAAAAAA/yWOQQ7CIBREz9I9fG3VC9gDaPAABgGTXyjfFNAYwt0VWM2bt5jMBOMJ9oP8BK605Q4fkG9RKsv+6p5RrpAFOcPmp295JYfqW2unwsIB8jkpa2K1nUqpfEnxlZoVJlDaVJuZyWuMSL4wT9rAEnbv8Vh/TMMSEPmWfMTVgOj5A7bFwKGjAAAA + Metadata: + aws:cdk:path: doc-example-glue-scenario-stack/CDKMetadata/Default + Condition: CDKMetadataAvailable +Outputs: + BucketName: + Value: + Ref: docexampleglue6E2F12E5 + RoleName: + Value: + Ref: AWSGlueServiceRoleDocExampleF579DC66 +Conditions: + CDKMetadataAvailable: + Fn::Or: + - Fn::Or: + - Fn::Equals: + - Ref: AWS::Region + - af-south-1 + - Fn::Equals: + - Ref: AWS::Region + - ap-east-1 + - Fn::Equals: + - Ref: AWS::Region + - ap-northeast-1 + - Fn::Equals: + - Ref: AWS::Region + - ap-northeast-2 + - Fn::Equals: + - Ref: AWS::Region + - ap-south-1 + - Fn::Equals: + - Ref: AWS::Region + - ap-southeast-1 + - Fn::Equals: + - Ref: AWS::Region + - ap-southeast-2 + - Fn::Equals: + - Ref: AWS::Region + - ca-central-1 + - Fn::Equals: + - Ref: AWS::Region + - cn-north-1 + - Fn::Equals: + - Ref: AWS::Region + - cn-northwest-1 + - Fn::Or: + - Fn::Equals: + - Ref: AWS::Region + - eu-central-1 + - Fn::Equals: + - Ref: AWS::Region + - eu-north-1 + - Fn::Equals: + - Ref: AWS::Region + - eu-south-1 + - Fn::Equals: + - Ref: AWS::Region + - eu-west-1 + - Fn::Equals: + - Ref: AWS::Region + - eu-west-2 + - Fn::Equals: + - Ref: AWS::Region + - eu-west-3 + - Fn::Equals: + - Ref: AWS::Region + - me-south-1 + - Fn::Equals: + - Ref: AWS::Region + - sa-east-1 + - Fn::Equals: + - Ref: AWS::Region + - us-east-1 + - Fn::Equals: + - Ref: AWS::Region + - us-east-2 + - Fn::Or: + - Fn::Equals: + - Ref: AWS::Region + - us-west-1 + - Fn::Equals: + - Ref: AWS::Region + - us-west-2 +Parameters: + BootstrapVersion: + Type: AWS::SSM::Parameter::Value + Default: /cdk-bootstrap/hnb659fds/version + Description: Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip] +Rules: + CheckBootstrapVersion: + Assertions: + - Assert: + Fn::Not: + - Fn::Contains: + - - "1" + - "2" + - "3" + - "4" + - "5" + - Ref: BootstrapVersion + AssertDescription: CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI. + diff --git a/swift/example_code/glue/scenario/test.sh b/swift/example_code/glue/scenario/test.sh new file mode 100644 index 00000000000..e69de29bb2d