diff --git a/beta/serverless-fleets/.gitignore b/beta/serverless-fleets/.gitignore index a4a65e157..b48719f77 100644 --- a/beta/serverless-fleets/.gitignore +++ b/beta/serverless-fleets/.gitignore @@ -6,8 +6,10 @@ data/taskstore/* data/output/docling/* data/output/wordcount/* data/output/inferencing/* +data/output/simulation/* !data/output/docling/.keep !data/output/wordcount/.keep !data/output/inferencing/.keep +!data/output/simulation/.keep */.DS_Store .DS_Store diff --git a/beta/serverless-fleets/README.md b/beta/serverless-fleets/README.md index 6e058f58c..fb81afab5 100644 --- a/beta/serverless-fleets/README.md +++ b/beta/serverless-fleets/README.md @@ -1,128 +1,50 @@ -# Serverless Fleets (beta) +# Simplify and optimize large-scale parallel computation with Serverless Fleets -Serverless Fleets is an new feature of IBM Cloud Code Engine. +As artificial intelligence continues to grow and demand for cloud-based solutions increases, the ability to run large-scale, compute-intensive workloads both quickly and efficiently has become critical. + +In this hands-on lab, you will deploy your first Serverless Fleet on IBM Code Engine—IBM’s strategic container platform designed to handle large-scale, compute-intensive workloads. + +Using both the intuitive graphical user interface and the command line, you will be guided step by step through the process. With just three clicks, you will have a Serverless Fleet up and running on IBM Cloud. **Table of Contents:** + +- [Key differentiators of Fleets](#key-differentiators-of-fleets) - [What is a fleets](#what-is-a-fleet) -- [Why using a fleets](#why-using-a-fleet) -- [The fleet concept](#the-fleet-concept) -- [Fleet specification](#fleet-specification) - [Architecture](#architecture) - [One Time Setup](#one-time-setup) - [Launch a Fleet](#launch-a-fleet) - [Launch a Fleet with GPUs](#launch-a-fleet-with-gpus) - [Launch a fleet with parallel tasks](#launch-a-fleet-with-parallel-tasks) - [Launch a fleet to count words of novels](#launch-a-fleet-to-count-words-of-novels) -- [Tutorials](#tutorials) +- [Tutorial: Docling](./tutorials/docling/README.md) +- [Tutorial: Batch Inferencing](./tutorials/inferencing/README.md) +- [Tutorial: Monte Carlo Simulation](./tutorials/simulation/README.md) - [HowTo](#howto) - [Troubleshooting](#troubleshooting) -## What is a fleet - -A fleet, also serverless fleet, is a Code Engine compute component that runs one or more instances of user code in order to complete its tasks. Instances run on workers which are automatically provisioned and de-provisioned based on the number and resource requirements of the instances. Fleets can provision any machine type which connect to Virtual Private Clouds (VPCs) and securely interoperate with user data and services there. - -## Why using a fleet -Fleets provide the following main key differentiators: -1. Any machine type including GPUs -2. Connected to the customers VPC network -3. Large scale parallel computing without limits on vCPU, Memory and duration -4. Dynamic task queuing to millions of tasks +## Key differentiators of Fleets -## The fleet concept +Fleets offer the following advantages: +1. Support for large-scale parallel computing tasks, with no limits on vCPU, memory, or task duration. +2. Automatic, dynamic scaling—from a single task to millions of tasks. +3. Consumption-based pricing: pay only for the resources you use, with no idle or fixed costs. +4. Fully managed service—no infrastructure administration required. +5. Broad machine type support, including GPU-enabled instances. +6. Seamless integration with your VPC network. -Fleets have three principal elements: tasks, instances and workers. +## What is a fleet ![](./images/prototype_concept.png) -### Tasks - -The tasks of a fleet represent the work that the fleet is intended to perform and are specified as part of the fleet specification at creation time. - -To perform that work, the fleet starts instances of user code on behalf of the tasks and maintains a representation of the current task statuses as instances are started and ending. Tasks change their initial “pending” status to “running” as soon as an instance is started on their behalf. The task status changes to “succeeded” if its instance process ends with a successful return code. If the instance ends unsuccessfully the associated task changes to status “failed” unless its maximum number of retries is not yet exhausted. In that case the task status is set back to “pending” so that a new instance can be started on behalf of the task. A special task status is “cancelled” which applies if the fleet is cancelled by user action. - -Once all tasks of a fleet have reached a final status, the fleet status also changes to a final status. The final fleet status is “succeeded” if all tasks have finished successfully, “failed” if at least one task failed and “cancelled” if the user has cancelled the fleet. Once a fleet has reached a final status, all instances have ended and all worker nodes are (being) de-provisioned - unless specific configuration settings change this behavior for debugging purposes. - -Through the tasks specification users can control the number of tasks, the order in which instances are started and which specific command and arguments are used to start an instance for a task. - -### Instances - -Instances of user code are started for the fleet’s tasks on top of worker nodes. Each instance is started on behalf of exactly one task, its associated task. Different instances always have different associated tasks. - -Fleets can work on many tasks in parallel by starting multiple instances concurrently. The maximum number of concurrent instances (max_scale) is part of the fleet’s specification. All instances are created with the same amount of vCPU and memory as per the fleet’s specification. - -Instances run user code as per the fleet’s code specification in combination with task parameters that allow for task-specific start commands and arguments. - -Instances terminate when the user code exits the instance process. The return code provided at that point signals whether the associated task was successfully completed (exit 0) or failed. The status of the associated task is updated accordingly and retries might be attempted as described in the preceding section on tasks. - -Instances might also be stopped if the fleet is cancelled with the “hard stop” option by user action or by exceeding the maximum execution time. - -### Workers - -Worker nodes are virtual machines automatically provisioned and de-provisioned based on the number and amount of resources required to run the fleet’s instances. +A fleet (also referred to as a serverless fleet) is a Code Engine compute resource that runs one or more instances of user code in parallel to process a large set of compute-intensive tasks. -Worker nodes are the basis for charging fleet resource consumption in terms of vCPU consumption, memory consumption and potential GPU uplifts. +Fleets can connect to Virtual Private Clouds (VPCs) to securely access user data and services. They provide dynamic task queuing, single-tenant isolation, and support for GPU workloads. -Users can influence the selection of worker node machine profiles by defining minimum requirements for eligible machine profiles or even specifying a certain one. +A fleet consists of a collection of worker nodes that automatically scale up or down based on resource requirements. Each instance runs on a worker node to complete a single task. When a task finishes, the worker node immediately starts the next task in the queue. This process continues until all tasks are completed, after which the worker nodes are automatically deprovisioned. -## Fleet specification +Like applications, jobs, and functions, fleets run within a Code Engine project. A project is a grouping of Code Engine resources within a specific IBM Cloud region. Projects are used to organize resources and manage access to entities such as configmaps, secrets, and persistent data stores. -Fleets run as soon as they are created so that “running a fleet” is the same operation as “creating a fleet”. Therefore the CLI provides `fleet run` and `fleet create` as synonyms. When creating a fleet the following aspects are specified - either explicitly or by default: -- name -- code -- tasks -- instance resources and scaling -- worker nodes -- connectivity -- environment variables (opt.) -- data store mounts (opt.) - -The default values are suitable in many cases so that running a fleet can be very easy and quick as shown in the examples section. - -### Name specification - -The fleet name identifies the fleet entity within the Code Engine project. It has to be a unique within fleets of the same Code Engine project, i.e. it might be the same as an app’s or job’s name in the same Code Engine project. - -### Tasks specification - -Fleets require at least one task and are designed to handle large number of tasks. There are two options to specify tasks: -- number of tasks: N -- tasks from file: - -Each tasks gets an index assigned from 0..N. The tasks index is provided as an environment variable `CE_TASK_INDEX` into the instance. - -In order to specify tasks in a file create a text file with line-wise definition of parameters in JSON syntax (according to JSONL standard). The task parameters "command" and "args" can be used to override of the command and arguments when starting an instance of user code on behalf of the task. If one or both of these parameters are specified their values are used instead of the respective definitions in the container image or in the fleet's code specification. For example, see [wordcount_commands.jsonl](./wordcount_commands.jsonl) - -### Code specification - -The fleet’s code determines what is run in one or more instances in order to work on tasks. The specification has two parts: the base specification defines a container image reference and optional command and arguments overrides. (This is the same for Code Engine apps and jobs.). In addition, fleets can override command and arguments in a task-specific way as described in the “Task specification” section. - -### Instance resources and scaling specifications - -vCPU and memory required by each instance can be specified and determines how many instances can fit/run on a fleet worker. In addition, the maximum number of concurrent instances (max_scale) can be specified. - -For example, if an instance requires 2 vCPU and 8 GB memory and a total of 100 instances should run concurrently, the fleet will provision a total of 200 vCPU and 800 GB memory. - -### Worker specifications - -Users can influence what machine profiles are used as worker nodes to different degrees. - -In the example above, if the user selects a bx2-8x32 worker profile, each worker can run 4 instances. Therefore a total of 25 workers will be provisioned. - -### Environment variables - -The instance will get the following environment provided by the system: -``` -CE_FLEET_VERSION=v1 -CE_REQUEST_ID=33af980d-8175-4925-85d0-0f0cf8812cb5 -CE_PROJECT_ID=e1501040-e56e-48b6-b9f0-1695908199bf -CE_FLEET_CONCURRENCY=1 -CE_TASK_ID=0 -CE_USER_MOUNT_POINT=/mnt/ce/data -CE_FLEET_KEEP_WORKER=false -CE_FLEET_ID=33af980d-8175-4925-85d0-0f0cf8812cb5 -CE_FLEET_IS_GPU=false -``` ## Architecture @@ -593,6 +515,8 @@ Download the results from the output COS bucket to `./data/output` ## Tutorials - [Tutorial: Docling](./tutorials/docling/README.md) +- [Tutorial: Inferencing](./tutorials/inferencing/README.md) +- [Tutorial: Simulation](./tutorials/simulation/README.md) ## HowTo diff --git a/beta/serverless-fleets/data/output/simulation/.keep b/beta/serverless-fleets/data/output/simulation/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/beta/serverless-fleets/images/docling-picture.png b/beta/serverless-fleets/images/docling-picture.png new file mode 100644 index 000000000..5fc165e10 Binary files /dev/null and b/beta/serverless-fleets/images/docling-picture.png differ diff --git a/beta/serverless-fleets/images/examples_simulation_flow.png b/beta/serverless-fleets/images/examples_simulation_flow.png index 3ff1518aa..a9f128859 100644 Binary files a/beta/serverless-fleets/images/examples_simulation_flow.png and b/beta/serverless-fleets/images/examples_simulation_flow.png differ diff --git a/beta/serverless-fleets/tutorials/docling/README.md b/beta/serverless-fleets/tutorials/docling/README.md index 53a5b5d49..12de70637 100644 --- a/beta/serverless-fleets/tutorials/docling/README.md +++ b/beta/serverless-fleets/tutorials/docling/README.md @@ -1,5 +1,7 @@ # Tutorial: Docling +![](../../images/docling-picture.png) + This tutorial provides a comprehensive guide on using Docling to convert PDFs into Markdown format using serverless fleets. It leverages cloud object storage for managing both the input PDFs and the resulting Markdown files. The process is streamlined using IBM’s Code Engine to build the Docling container, which is then pushed to a container registry. Users can run a serverless fleet, which autonomously spawns workers to run the Docling container for efficient, scalable conversion tasks. Key steps covered in the Tutorial: diff --git a/beta/serverless-fleets/tutorials/simulation/.ceignore b/beta/serverless-fleets/tutorials/simulation/.ceignore new file mode 100644 index 000000000..5ceb3864c --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/.ceignore @@ -0,0 +1 @@ +venv diff --git a/beta/serverless-fleets/tutorials/simulation/.gitignore b/beta/serverless-fleets/tutorials/simulation/.gitignore new file mode 100644 index 000000000..5ceb3864c --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/.gitignore @@ -0,0 +1 @@ +venv diff --git a/beta/serverless-fleets/tutorials/simulation/Dockerfile b/beta/serverless-fleets/tutorials/simulation/Dockerfile new file mode 100644 index 000000000..49c41108e --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.11-slim-bookworm + +ENV GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no" + +RUN apt-get update \ + && apt-get clean + +WORKDIR /app + +COPY simulate.py /app/simulate.py +COPY requirements.txt /app/requirements.txt + +RUN python3 -m ensurepip && pip install --no-cache-dir -r /app/requirements.txt + +# Reset the entrypoint +ENTRYPOINT [] + +CMD ["python3", "simulate.py"] + diff --git a/beta/serverless-fleets/tutorials/simulation/README.md b/beta/serverless-fleets/tutorials/simulation/README.md new file mode 100644 index 000000000..726a01810 --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/README.md @@ -0,0 +1,257 @@ +# Tutorial: Monte Carlo Simulation + +This tutorial provides a comprehensive guide of running a simple stock price simulation from local source code using serverless fleets. The example, performs a Monte Carlo Simulation to calculate the Value-at-Risk of 24 stocks. As an input the user provides the source code and the list of 24 commands for each stock ticker. Code Engine *builds* are used to build the container image. The *serverless fleet* will autonomously spawns workers to run the simulation. The result is written to a Cloud Object Storage bucket. + +Key steps covered in the Tutorial: +1. Containerization with Code Engine: Build the simulation container and push it to a registry for deployment. +3. Run a fleet of workers that automatically runs the container, ensuring scalability and efficiency. +4. Download the results from COS + + +![](../../images/examples_simulation_flow.png) + +> Note: The tutorial assumes that you have created the fleet sandbox using the fully automated approach which creates the rclone environment as well as the upload/download scripts. If that's not the case, you would need to upload the PDFs and download the results using the COS CLI or other means. + +## Steps + + +### Step 1 - Build and Push the container registry + +Build the container image using Code Engine's build capabilities by running the following command in the `tutorials/simulation` directory. + +If you're interested review the code, by looking at the [simulate.py](./simulate.py), which contains a simple method that downloads the last year stock prices of a stock ticker, performs 100k simulations and writes the VaR at a csv file. It receives the stock ticker and output directory as input arguments. +``` +cat simulate.py +``` + +Now, run the build script to run a Code Engine build to build a container image using and push it to the container registry + +``` +cd tutorials/simulation +./build +``` + +### Step 2 - Prepare the tasks + +In this tutorial we use the `--tasks-from-file commands.jsonl` option to submit the tasks. Therefore we have prepared a file in [jsonfiles](https://jsonlines.org/) format which contains 1 task per line. Each line specifies command and arguments for this task. Let's review the file in the `tutorials/simulation` directory: + +``` +cat commands.jsonl +``` + + +
+ Output + +``` +➜ simulation cat commands.jsonl + { "cmds": ["python3"], "args": ["simulate.py", "AKAM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "MO", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AMZN", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AMGN", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AAPL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "T", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "BA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "CAT", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "CVX", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "DIS", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "KO", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "DELL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "F", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "INTC", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "IBM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "MSFT", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "NFLX", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "NVDA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "ORCL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "QCOM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "X", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "VZ", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "V", "/output"]} +``` + +
+
+ + +### Step 3 - Run the Fleet + +Now run the fleet to process the 24 stock tickers. In this tutorial we use the `--tasks-from-file commands.jsonl` to specify the tasks. Each task will get 1 CPU and 4 GB memory. We specify a specific machine profile `--worker-profile mx2-4x32` and configure `--max-scale 24` to run all 24 simulations in parallel. Therefore, the system is deploying 6 workers, each running 4 instances concurrently, e.g. each worker is running 4 simulations at a point in time. +``` +./run +``` + + +
+ Output + +``` +➜ simulation ./run +ibmcloud code-engine beta fleet create --name fleet-c042e88d-1 + --image private.br.icr.io/ce--fleet-simulation/simulation + --registry-secret ce-auto-icr-private-br-sao + --worker-profile mx2-4x32 + --tasks-from-local-file commands.jsonl + --cpu 1 + --memory 8G + --max-scale 24 + --mount-data-store /output=fleet-output-store:/simulation +Successfully created fleet with name 'fleet-c042e88d-1' and ID '630569df-c6c0-44dc-a376-e2f5d8d7aad8' +Run 'ibmcloud ce beta fleet get --id 630569df-c6c0-44dc-a376-e2f5d8d7aad8' to check the fleet status. +Run 'ibmcloud ce beta fleet worker list --fleet-id 630569df-c6c0-44dc-a376-e2f5d8d7aad8' to retrieve a list of provisioned workers. +Run 'ibmcloud ce beta fleet task list --fleet-id 630569df-c6c0-44dc-a376-e2f5d8d7aad8' to retrieve a list of tasks. +OK +``` +
+
+ +Review the fleet +``` +ibmcloud code-engine beta fleet get --id +``` + +
+ Output + +``` +➜ simulation ibmcloud ce beta fleet get --id 630569df-c6c0-44dc-a376-e2f5d8d7aad8 +Getting fleet '630569df-c6c0-44dc-a376-e2f5d8d7aad8'... +OK + +Name: fleet-c042e88d-1 +ID: 630569df-c6c0-44dc-a376-e2f5d8d7aad8 +Status: pending +Created: 5s +Project region: br-sao +Project name: fleetlab-dev--ce-project + +Tasks status: + Failed: 0 + Canceled: 0 + Successful: 0 + Running: 0 + Pending: 24 + Total: 24 + +Code: + Container image reference: private.br.icr.io/ce--fleet-simulation/simulation + Registry access secret: ce-auto-icr-private-br-sao + +Tasks specification: + Task state store: fleet-task-store + Data store JSON reference: fleet-task-store + Data store object path: /ce/2c76a9f0-507e-472b-84be-81efe50403f8/fleet-input/80d4f857-62f9-4292-9672-364109ae4aa6.jsonl + +Resources and scaling: + CPU per instance: 1 + Memory per instance: 8G + Preferred worker profile: mx2-4x32 + Max number of instances: 24 + Max retries per task: 3 + +Network placement: + Subnet CRN 0: crn:v1:bluemix:public:is:br-sao-1:a/327016f62a9544c18e7efdd4213297dd::subnet:02t7-61ad2d36-695c-41b2-8bd1-38ee926cb94a +``` +
+
+ + + +Verify that the machines are starting +``` +ibmcloud code-engine beta fleet worker list --fleet-id +``` + +
+ Output + +``` +➜ ibmcloud ce beta fleet worker list --fleet-id 709f5fee-59ca-41b3-b518-9e5f665e4d78 +Listing serverless fleet workers... +OK + +Name ID Status Profile IP Zone Version +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-0 885a775f-9918-4da5-b9ee-40bb3dc3f02e initializing mx2-4x32 10.250.0.24 br-sao-1 v0.0.78 +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-1 554c2232-f7fe-4081-babc-c7490e90742a initializing mx2-4x32 10.250.0.23 br-sao-1 v0.0.78 +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-2 9b7f8195-10c1-43d5-854d-09043608f4d9 initializing mx2-4x32 10.250.0.25 br-sao-1 v0.0.78 +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-3 033c0bbf-f601-4df5-a53e-76b9fe7ef255 initializing mx2-4x32 10.250.0.22 br-sao-1 v0.0.78 +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-4 c21b4a8b-7468-415c-905e-7cafa8646222 initializing mx2-4x32 10.250.0.21 br-sao-1 v0.0.78 +fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-5 1113a857-9880-4193-9e6e-d62fc9c66a0f initializing mx2-4x32 10.250.0.26 br-sao-1 v0.0.78 +``` +
+
+ +Observe the tasks: + +``` +ibmcloud code-engine beta fleet task list --fleet-id +``` + +
+ Output + +``` +➜ serverless-fleets ibmcloud ce beta fleet task list --fleet-id 709f5fee-59ca-41b3-b518-9e5f665e4d78 +Listing serverless fleet tasks... +OK + +Index ID Status Result code Worker name +000-00000-00000000000000000000 767658c4-a311-5cdb-86ee-1f81d5a0546d running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-4 +000-00000-00000000000000000001 8b6cf1f7-839e-56e0-b5ba-5cf76b950774 running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-3 +000-00000-00000000000000000002 53d21f36-a46a-5caa-a3ec-ad37c33bf9ac running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-3 +000-00000-00000000000000000003 5b4426d6-bb15-5c39-b38a-bd42bbc402ca running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-4 +000-00000-00000000000000000004 9bc3ab9b-36a9-5bf1-aba4-6f594ca520ed running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-0 +000-00000-00000000000000000005 b21f26d8-bda3-5950-b109-f9eac16381ec running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-5 +000-00000-00000000000000000006 78d47365-3ced-59bb-b895-955c52fb4c28 running fleet-709f5fee-59ca-41b3-b518-9e5f665e4d78-1 +... +``` +
+
+ +### Step 4 - Download results + +Download the results from the COS by running the following command in the root directory: +``` +./download +``` + +You can find the results under +``` +ls -l data/output/simulation/ticker_* +``` + + +
+ Output + +``` +➜ serverless-fleets ls -l data/output/simulation/ticker_* +-rw-r--r-- 1 jeremiaswerner staff 31 Sep 10 15:16 ticker_AA.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_AAPL.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_AKAM.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_AMGN.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_AMZN.result +-rw-r--r-- 1 jeremiaswerner staff 31 Sep 10 15:16 ticker_BA.result +-rw-r--r-- 1 jeremiaswerner staff 32 Sep 10 15:16 ticker_CAT.result +-rw-r--r-- 1 jeremiaswerner staff 32 Sep 10 15:16 ticker_CVX.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_DELL.result +-rw-r--r-- 1 jeremiaswerner staff 32 Sep 10 15:16 ticker_DIS.result +-rw-r--r-- 1 jeremiaswerner staff 30 Sep 10 15:16 ticker_F.result +-rw-r--r-- 1 jeremiaswerner staff 32 Sep 10 15:16 ticker_IBM.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_INTC.result +-rw-r--r-- 1 jeremiaswerner staff 31 Sep 10 15:16 ticker_KO.result +-rw-r--r-- 1 jeremiaswerner staff 31 Sep 10 15:16 ticker_MO.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_MSFT.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_NFLX.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_NVDA.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_ORCL.result +-rw-r--r-- 1 jeremiaswerner staff 33 Sep 10 15:16 ticker_QCOM.result +-rw-r--r-- 1 jeremiaswerner staff 30 Sep 10 15:16 ticker_T.result +-rw-r--r-- 1 jeremiaswerner staff 30 Sep 10 15:16 ticker_V.result +-rw-r--r-- 1 jeremiaswerner staff 31 Sep 10 15:16 ticker_VZ.result +-rw-r--r-- 1 jeremiaswerner staff 27 Sep 10 15:16 ticker_X.result +``` +
+
+ diff --git a/beta/serverless-fleets/tutorials/simulation/build b/beta/serverless-fleets/tutorials/simulation/build new file mode 100755 index 000000000..b7f56c725 --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/build @@ -0,0 +1,24 @@ +#!/bin/sh + +#!/bin/bash + +echo "Cleaning up previous images ..." +IMAGE=$(ibmcloud cr images | grep "ce--fleet-simulation" | awk '{print $1}') +if [[ "$IMAGE" != "" ]]; then + ibmcloud cr image-rm $(ibmcloud cr images | grep "ce--fleet-simulation" | awk '{print $1}') +fi + +# getting the automatically created registry secret, e.g. ce-auto-icr-private- +REGION=$(ibmcloud target -o json | jq -r '.region.name') +REGISTRY_SECRET_NAME="ce-auto-icr-private-$REGION" +REGISTRY_HOST=$(ibmcloud ce secret get -n $REGISTRY_SECRET_NAME -o json | jq -r ".data.server") + +echo "Building new image $REGISTRY_HOST/ce--fleet-simulation-${uuid}/simulation:latest from ./src " +echo "This takes about 1-2 minutes ..." + +uuid=$(uuidgen | tr '[:upper:]' '[:lower:]' | awk -F- '{print $1}') + +ibmcloud ce buildrun submit --source . --strategy dockerfile --image $REGISTRY_HOST/ce--fleet-simulation/simulation:latest --registry-secret $REGISTRY_SECRET_NAME --name ce--fleet-simulation-build-${uuid} --size medium --timeout 300 + +ibmcloud ce buildrun logs -f -n ce--fleet-simulation-build-${uuid} + diff --git a/beta/serverless-fleets/tutorials/simulation/commands.jsonl b/beta/serverless-fleets/tutorials/simulation/commands.jsonl new file mode 100644 index 000000000..3ec9c6aba --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/commands.jsonl @@ -0,0 +1,24 @@ + { "cmds": ["python3"], "args": ["simulate.py", "AKAM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "MO", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AMZN", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AMGN", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "AAPL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "T", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "BA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "CAT", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "CVX", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "DIS", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "KO", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "DELL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "F", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "INTC", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "IBM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "MSFT", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "NFLX", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "NVDA", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "ORCL", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "QCOM", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "X", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "VZ", "/output"]} + { "cmds": ["python3"], "args": ["simulate.py", "V", "/output"]} diff --git a/beta/serverless-fleets/tutorials/simulation/local b/beta/serverless-fleets/tutorials/simulation/local new file mode 100755 index 000000000..c34bc088c --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/local @@ -0,0 +1,4 @@ +python3 -m venv venv +source ./venv/bin/activate +python3 -m pip install --upgrade pip +pip3 install -r requirements.txt diff --git a/beta/serverless-fleets/tutorials/simulation/requirements.txt b/beta/serverless-fleets/tutorials/simulation/requirements.txt new file mode 100644 index 000000000..e77cf5fe0 --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/requirements.txt @@ -0,0 +1,3 @@ +matplotlib +yfinance +numpy diff --git a/beta/serverless-fleets/tutorials/simulation/run b/beta/serverless-fleets/tutorials/simulation/run new file mode 100755 index 000000000..3aba34f09 --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/run @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +uuid=$(uuidgen | tr '[:upper:]' '[:lower:]' | awk -F- '{print $1}') + +IMAGE=$(ibmcloud cr images | grep "ce--fleet-simulation" | awk '{print $1}') + +if [ -z "${IMAGE}" ]; then + echo "no image found. pls build a simulation image with ./build.sh" + exit -1 +fi + +MAX_SCALE=24 +PROFILE="mx2-4x32" +CPU=1 +MEMORY=8G + +# getting the automatically created registry secret, e.g. ce-auto-icr-private- +REGION=$(ibmcloud target -o json | jq -r '.region.name') +REGISTRY_SECRET_NAME="ce-auto-icr-private-$REGION" + +PRIVATE_IMAGE="private.$IMAGE" + +echo ibmcloud code-engine beta fleet create --name "fleet-${uuid}-1" +echo " "--image $PRIVATE_IMAGE +echo " "--registry-secret $REGISTRY_SECRET_NAME +echo " "--worker-profile $PROFILE +echo " "--tasks-from-local-file commands.jsonl +echo " "--cpu $CPU +echo " "--memory $MEMORY +echo " "--max-scale $MAX_SCALE +echo " "--mount-data-store /output=fleet-output-store:/simulation + +ibmcloud code-engine beta fleet create --name "fleet-${uuid}-1" \ +--image $PRIVATE_IMAGE \ +--registry-secret $REGISTRY_SECRET_NAME \ +--max-scale $MAX_SCALE \ +--tasks-from-local-file commands.jsonl \ +--cpu $CPU \ +--memory $MEMORY \ +--worker-profile $PROFILE \ +--tasks-state-store fleet-task-store \ +--mount-data-store /output=fleet-output-store:/simulation + diff --git a/beta/serverless-fleets/tutorials/simulation/simulate.py b/beta/serverless-fleets/tutorials/simulation/simulate.py new file mode 100644 index 000000000..d5a9f7c03 --- /dev/null +++ b/beta/serverless-fleets/tutorials/simulation/simulate.py @@ -0,0 +1,54 @@ +import sys +import os +import numpy as np +import yfinance as yf + +def monte_carlo_simulation(ticker): + # Fetch historical data for a stock + data = yf.download(ticker, start="2024-01-01", end="2025-01-01", auto_adjust=True, progress=False) + returns = data['Close'].pct_change().dropna() + + # Simulate future returns using Monte Carlo + num_simulations = 100000 + simulation_horizon = 252 # Number of trading days in a year + stddev = np.std(returns, axis=0) + simulated_returns = np.random.normal(np.mean(returns, axis=0), stddev, (simulation_horizon, num_simulations)) + + # Calculate the simulated portfolio values + initial_investment = 1000000 # $1,000,000 + portfolio_values = initial_investment * np.exp(np.cumsum(simulated_returns, axis=0)) + + # Calculate the portfolio returns + portfolio_returns = portfolio_values[-1] / portfolio_values[0] - 1 + + # Calculate the VaR at 99.5% confidence level + confidence_level = 0.995 + VaR = np.percentile(portfolio_returns, (1 - confidence_level) * 100) + return (ticker, VaR, stddev.iloc[0]) + +def print_usage(): + print("usage: python3 simulate.py ") + print("") + print(" TICKER : stock ticker" ) + print(" RESULT_FOLDER : folder of the results") + print("") + sys.exit(-1) + + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print_usage() + + TICKER = sys.argv[1] + RESULT_FOLDER = sys.argv[2] + + (ticker, VaR, stddev) = monte_carlo_simulation(TICKER) + + print("VaR of ticker %s: %s (%s)" % (ticker, VaR, stddev) ) + + filename="%s/ticker_%s.result" % (RESULT_FOLDER,TICKER) + with open(filename, "w+") as f: + f.write("Ticker,VaR,STDEV\n") + f.write("%s,%.2f,%.2f\n" % (ticker,VaR,stddev)) +