|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * |
| 4 | + * This source code is licensed under the MIT license found in the |
| 5 | + * LICENSE file in the root directory of this source tree. |
| 6 | + */ |
| 7 | + |
| 8 | +// Jupyter notebook parsing support. |
| 9 | +// |
| 10 | +// This module extracts Python code from `.ipynb` files (Jupyter notebooks) |
| 11 | +// and converts them into a single Python source file that can be analyzed |
| 12 | +// by the type checker. |
| 13 | + |
| 14 | +use anyhow::Context; |
| 15 | +use anyhow::Result; |
| 16 | +use serde::Deserialize; |
| 17 | + |
| 18 | +/// Represents a Jupyter notebook cell |
| 19 | +#[derive(Debug, Deserialize)] |
| 20 | +struct NotebookCell { |
| 21 | + cell_type: String, |
| 22 | + source: NotebookSource, |
| 23 | +} |
| 24 | + |
| 25 | +/// Source can be either a string or an array of strings |
| 26 | +#[derive(Debug, Deserialize)] |
| 27 | +#[serde(untagged)] |
| 28 | +enum NotebookSource { |
| 29 | + String(String), |
| 30 | + Array(Vec<String>), |
| 31 | +} |
| 32 | + |
| 33 | +/// Minimal representation of a Jupyter notebook |
| 34 | +#[derive(Debug, Deserialize)] |
| 35 | +struct Notebook { |
| 36 | + cells: Vec<NotebookCell>, |
| 37 | +} |
| 38 | + |
| 39 | +/// Extracts Python code from a Jupyter notebook JSON string. |
| 40 | +/// |
| 41 | +/// This function: |
| 42 | +/// - Parses the notebook JSON |
| 43 | +/// - Extracts only code cells (ignoring markdown cells) |
| 44 | +/// - Concatenates all code into a single Python source string |
| 45 | +/// - Adds cell markers as comments for debugging |
| 46 | +/// |
| 47 | +/// # Arguments |
| 48 | +/// * `content` - The raw JSON content of a `.ipynb` file |
| 49 | +/// |
| 50 | +/// # Returns |
| 51 | +/// A single Python source string containing all code cells, or an error if parsing fails |
| 52 | +pub fn extract_python_from_notebook(content: &str) -> Result<String> { |
| 53 | + let notebook: Notebook = |
| 54 | + serde_json::from_str(content).context("Failed to parse notebook JSON")?; |
| 55 | + |
| 56 | + let mut python_code = String::new(); |
| 57 | + let mut code_cell_count = 0; |
| 58 | + |
| 59 | + for cell in notebook.cells.iter() { |
| 60 | + if cell.cell_type == "code" { |
| 61 | + code_cell_count += 1; |
| 62 | + // Add a comment marker for each cell |
| 63 | + python_code.push_str(&format!("# Cell {}\n", code_cell_count)); |
| 64 | + |
| 65 | + // Extract the source code |
| 66 | + match &cell.source { |
| 67 | + NotebookSource::String(s) => { |
| 68 | + python_code.push_str(s.as_str()); |
| 69 | + } |
| 70 | + NotebookSource::Array(lines) => { |
| 71 | + for line in lines { |
| 72 | + python_code.push_str(line.as_str()); |
| 73 | + } |
| 74 | + } |
| 75 | + } |
| 76 | + |
| 77 | + // Add spacing between cells |
| 78 | + if !python_code.ends_with('\n') { |
| 79 | + python_code.push('\n'); |
| 80 | + } |
| 81 | + python_code.push('\n'); |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + Ok(python_code) |
| 86 | +} |
| 87 | + |
| 88 | +#[cfg(test)] |
| 89 | +mod tests { |
| 90 | + use super::*; |
| 91 | + |
| 92 | + #[test] |
| 93 | + fn test_extract_python_from_notebook() { |
| 94 | + let notebook_json = r##"{ |
| 95 | + "cells": [ |
| 96 | + { |
| 97 | + "cell_type": "code", |
| 98 | + "source": ["def hello():\n", " return 'world'\n"] |
| 99 | + }, |
| 100 | + { |
| 101 | + "cell_type": "markdown", |
| 102 | + "source": ["This is a markdown cell"] |
| 103 | + }, |
| 104 | + { |
| 105 | + "cell_type": "code", |
| 106 | + "source": "x = 5" |
| 107 | + } |
| 108 | + ] |
| 109 | + }"##; |
| 110 | + |
| 111 | + let result = extract_python_from_notebook(notebook_json).unwrap(); |
| 112 | + |
| 113 | + assert!(result.contains("# Cell 1")); |
| 114 | + assert!(result.contains("def hello():")); |
| 115 | + assert!(result.contains("return 'world'")); |
| 116 | + assert!(!result.contains("This is a markdown cell")); |
| 117 | + assert!(result.contains("# Cell 2")); |
| 118 | + assert!(result.contains("x = 5")); |
| 119 | + } |
| 120 | + |
| 121 | + #[test] |
| 122 | + fn test_extract_python_from_empty_notebook() { |
| 123 | + let notebook_json = r#"{"cells": []}"#; |
| 124 | + let result = extract_python_from_notebook(notebook_json).unwrap(); |
| 125 | + assert_eq!(result, ""); |
| 126 | + } |
| 127 | + |
| 128 | + #[test] |
| 129 | + fn test_extract_python_with_invalid_json() { |
| 130 | + let invalid_json = "not valid json"; |
| 131 | + let result = extract_python_from_notebook(invalid_json); |
| 132 | + assert!(result.is_err()); |
| 133 | + } |
| 134 | +} |
0 commit comments