Skip to content

Commit f9d07ac

Browse files
mbertrandmbertrand
authored
Refactor price assignment for mitxonline (#2550)
1 parent e9105dc commit f9d07ac

File tree

4 files changed

+39
-79
lines changed

4 files changed

+39
-79
lines changed

‎learning_resources/etl/mitxonline.py‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -153,18 +153,23 @@ def extract_courses():
153153
return []
154154

155155

156-
def parse_program_prices(program_data: dict) -> list[dict]:
157-
"""Return a list of unique prices for a program"""
158-
prices = [program_data.get("current_price") or 0.00]
159-
price_string = parse_page_attribute(program_data, "price")
160-
if price_string:
161-
prices.extend(
162-
[
163-
float(price.replace(",", ""))
164-
for price in re.findall(r"[\d\.,]+", price_string)
165-
]
156+
def parse_prices(parent_data: dict) -> list[dict]:
157+
"""
158+
Return a list of unique prices for a course/program.
159+
$0.00 (free) is always included for the non-certificate option.
160+
Other prices come from the parent course/program's min_price & max_price fields.
161+
"""
162+
free_price_str = "0.00"
163+
return [
164+
transform_price(price)
165+
for price in sorted(
166+
{
167+
Decimal(free_price_str),
168+
Decimal(parent_data.get("min_price") or free_price_str),
169+
Decimal(parent_data.get("max_price") or free_price_str),
170+
}
166171
)
167-
return [transform_price(Decimal(price)) for price in sorted(set(prices))]
172+
]
168173

169174

170175
def parse_departments(departments_data: list[dict or str]) -> list[str]:
@@ -226,22 +231,7 @@ def _transform_run(course_run: dict, course: dict) -> dict:
226231
),
227232
"description": clean_data(parse_page_attribute(course_run, "description")),
228233
"image": _transform_image(course_run),
229-
"prices": [
230-
transform_price(price)
231-
for price in sorted(
232-
{
233-
Decimal("0.00"),
234-
*[
235-
Decimal(price)
236-
for price in [
237-
product.get("price")
238-
for product in course_run.get("products", [])
239-
]
240-
if price is not None
241-
],
242-
}
243-
)
244-
],
234+
"prices": parse_prices(course),
245235
"instructors": [
246236
{"full_name": instructor["name"]}
247237
for instructor in parse_page_attribute(course, "instructors", is_list=True)
@@ -418,7 +408,7 @@ def transform_programs(programs: list[dict]) -> list[dict]:
418408
"description": clean_data(
419409
parse_page_attribute(program, "description")
420410
),
421-
"prices": parse_program_prices(program),
411+
"prices": parse_prices(program),
422412
"status": RunStatus.current.value
423413
if parse_page_attribute(program, "page_url")
424414
else RunStatus.archived.value,

‎learning_resources/etl/mitxonline_test.py‎

Lines changed: 13 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
# pylint: disable=redefined-outer-name
66
from datetime import datetime
7-
from decimal import Decimal
87
from unittest.mock import ANY
98
from urllib.parse import parse_qs, urljoin, urlparse
109

@@ -30,7 +29,7 @@
3029
extract_programs,
3130
parse_certificate_type,
3231
parse_page_attribute,
33-
parse_program_prices,
32+
parse_prices,
3433
transform_courses,
3534
transform_programs,
3635
transform_topics,
@@ -172,7 +171,7 @@ def test_mitxonline_transform_programs(
172171
"published": bool(
173172
program_data.get("page", {}).get("page_url", None) is not None
174173
),
175-
"prices": parse_program_prices(program_data),
174+
"prices": parse_prices(program_data),
176175
"image": _transform_image(program_data),
177176
"title": program_data["title"],
178177
"description": clean_data(
@@ -262,25 +261,7 @@ def test_mitxonline_transform_programs(
262261
course_run_data["is_enrollable"]
263262
and course_data["page"]["live"]
264263
),
265-
"prices": sorted(
266-
[
267-
{"amount": Decimal(i), "currency": CURRENCY_USD}
268-
for i in {
269-
0.00,
270-
*[
271-
price
272-
for price in [
273-
product.get("price")
274-
for product in course_run_data.get(
275-
"products", []
276-
)
277-
]
278-
if price is not None
279-
],
280-
}
281-
],
282-
key=lambda x: x["amount"],
283-
),
264+
"prices": parse_prices(course_data),
284265
"instructors": [
285266
{"full_name": instructor["name"]}
286267
for instructor in parse_page_attribute(
@@ -399,25 +380,7 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data):
399380
"published": bool(
400381
course_run_data["is_enrollable"] and course_data["page"]["live"]
401382
),
402-
"prices": sorted(
403-
[
404-
{"amount": Decimal(i), "currency": CURRENCY_USD}
405-
for i in {
406-
0.00,
407-
*[
408-
price
409-
for price in [
410-
product.get("price")
411-
for product in course_run_data.get(
412-
"products", []
413-
)
414-
]
415-
if price is not None
416-
],
417-
}
418-
],
419-
key=lambda x: x["amount"],
420-
),
383+
"prices": parse_prices(course_data),
421384
"instructors": [
422385
{"full_name": instructor["name"]}
423386
for instructor in parse_page_attribute(
@@ -536,19 +499,18 @@ def test_program_run_start_date_value( # noqa: PLR0913
536499

537500

538501
@pytest.mark.parametrize(
539-
("current_price", "page_price", "expected"),
502+
("min_price", "max_price", "expected"),
540503
[
541-
(0, "100", [0, 100]),
542-
(None, "$100 - $1,000", [0, 100, 1000]),
543-
(99.99, "$99.99 - $3,000,000", [99.99, 3000000]),
544-
(9.99, "$99.99 per course", [9.99, 99.99]),
545-
(100, "varies from $29-$129", [100, 29, 129]),
504+
(None, 100, [0, 100]),
505+
(100, 1000, [0, 100, 1000]),
506+
(99.99, 3000, [0.00, 99.99, 3000]),
507+
(9.99, None, [0, 9.99]),
546508
],
547509
)
548-
def test_parse_prices(current_price, page_price, expected):
549-
"""Test that the prices are correctly parsed from the page data"""
550-
program_data = {"current_price": current_price, "page": {"price": page_price}}
551-
assert parse_program_prices(program_data) == sorted(
510+
def test_parse_prices(min_price, max_price, expected):
511+
"""Test that the prices are correctly parsed from the parent data"""
512+
program_data = {"min_price": min_price, "max_price": max_price}
513+
assert parse_prices(program_data) == sorted(
552514
[{"amount": float(price), "currency": CURRENCY_USD} for price in expected],
553515
key=lambda x: x["amount"],
554516
)

‎test_json/mitxonline_courses.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
"name": "Mathematics"
4545
}
4646
],
47+
"min_price": 123,
48+
"max_price": 456,
4749
"certificate_type": "Certificate of Completion",
4850
"required_prerequisites": true,
4951
"duration": "14 weeks",
@@ -147,6 +149,8 @@
147149
}
148150
]
149151
},
152+
"min_price": null,
153+
"max_price": 456,
150154
"programs": null,
151155
"topics": [
152156
{
@@ -218,6 +222,8 @@
218222
}
219223
]
220224
},
225+
"min_price": 123,
226+
"max_price": null,
221227
"programs": null,
222228
"topics": [
223229
{

‎test_json/mitxonline_programs.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@
8888
"effort": "7 hrs/wk",
8989
"price": "$175"
9090
},
91+
"min_price": 1230,
92+
"max_price": 4560,
9193
"program_type": "Series",
9294
"certificate_type": "Certificate of Completion",
9395
"departments": [

0 commit comments

Comments
 (0)