diff --git a/.yarnrc.yml b/.yarnrc.yml index 489bae8418..9b9bab6ac5 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -21,3 +21,4 @@ supportedArchitectures: libc: - current - glibc +enableScripts: false diff --git a/RELEASE.rst b/RELEASE.rst index 73a31c000c..1dd5e310b3 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,15 @@ Release Notes ============= +Version 0.45.1 +-------------- + +- allow multiple problem files (#2512) +- Celery task tweaks (#2545) +- feat: installs granian as a replacement process manager for production / hosted envs. (#2544) +- security: disable yarn postinstall scripts (#2543) +- Separate Login and Signup URLs (again) (#2535) + Version 0.45.0 (Released September 24, 2025) -------------- diff --git a/authentication/views.py b/authentication/views.py index 79b44ee7fd..91a8a1e91b 100644 --- a/authentication/views.py +++ b/authentication/views.py @@ -15,25 +15,26 @@ log = logging.getLogger(__name__) -def get_redirect_url(request): +def get_redirect_url(request, param_names): """ Get the redirect URL from the request. Args: request: Django request object + param_names: Names of the GET parameter or cookie to look for the redirect URL; + first match will be used. Returns: str: Redirect URL """ - next_url = request.GET.get("next") or request.COOKIES.get("next") - return ( - next_url - if next_url - and url_has_allowed_host_and_scheme( + for param_name in param_names: + next_url = request.GET.get(param_name) or request.COOKIES.get(param_name) + if next_url and url_has_allowed_host_and_scheme( next_url, allowed_hosts=settings.ALLOWED_REDIRECT_HOSTS - ) - else "/app" - ) + ): + return next_url + + return "/app" class CustomLogoutView(View): @@ -51,7 +52,7 @@ def get( GET endpoint reached after logging a user out from Keycloak """ user = getattr(request, "user", None) - user_redirect_url = get_redirect_url(request) + user_redirect_url = get_redirect_url(request, ["next"]) if user and user.is_authenticated: logout(request) if request.META.get(ApisixUserMiddleware.header): @@ -77,7 +78,8 @@ def get( """ GET endpoint for logging a user in. """ - redirect_url = get_redirect_url(request) + redirect_url = get_redirect_url(request, ["next"]) + signup_redirect_url = get_redirect_url(request, ["signup_next", "next"]) if not request.user.is_anonymous: profile = request.user.profile @@ -104,12 +106,14 @@ def get( redirect_url = urljoin( settings.APP_BASE_URL, f"/dashboard/organization/{org_slug}" ) - elif ( - not profile.has_logged_in - and request.GET.get("skip_onboarding", "0") == "0" - ): - params = urlencode({"next": redirect_url}) - redirect_url = f"{settings.MITOL_NEW_USER_LOGIN_URL}?{params}" + # first-time non-org users + elif not profile.has_logged_in: + if request.GET.get("skip_onboarding", "0") == "0": + params = urlencode({"next": signup_redirect_url}) + redirect_url = f"{settings.MITOL_NEW_USER_LOGIN_URL}?{params}" + profile.save() + else: + redirect_url = signup_redirect_url if not profile.has_logged_in: profile.has_logged_in = True diff --git a/authentication/views_test.py b/authentication/views_test.py index ac88479508..c88a610335 100644 --- a/authentication/views_test.py +++ b/authentication/views_test.py @@ -7,24 +7,44 @@ import pytest from django.test import RequestFactory +from django.urls import reverse +from django.utils.http import urlencode from authentication.views import CustomLoginView, get_redirect_url @pytest.mark.parametrize( - ("next_url", "allowed"), + ("param_names", "expected_redirect"), [ - ("/app", True), - ("http://open.odl.local:8062/search", True), - ("http://open.odl.local:8069/search", False), - ("https://ocw.mit.edu", True), - ("https://fake.fake.edu", False), + (["exists-a"], "/url-a"), + (["exists-b"], "/url-b"), + (["exists-a", "exists-b"], "/url-a"), + (["exists-b", "exists-a"], "/url-b"), + (["not-exists-x", "exists-a"], "/url-a"), + (["not-exists-x", "not-exists-y"], "/app"), + # With disallowed hosts in the params + (["disallowed-1"], "/app"), + (["not-exists-x", "disallowed-1"], "/app"), + (["disallowed-1", "exists-a"], "/url-a"), + (["allowed-2"], "https://good.com/url-2"), ], ) -def test_custom_login(mocker, next_url, allowed): +def test_get_redirect_url(mocker, param_names, expected_redirect): """Next url should be respected if host is allowed""" - mock_request = mocker.MagicMock(GET={"next": next_url}) - assert get_redirect_url(mock_request) == (next_url if allowed else "/app") + GET = { + "exists-a": "/url-a", + "exists-b": "/url-b", + "exists-c": "/url-c", + "disallowed-a": "https://malicious.com/url-1", + "allowed-2": "https://good.com/url-2", + } + mocker.patch( + "authentication.views.settings.ALLOWED_REDIRECT_HOSTS", + ["good.com"], + ) + + mock_request = mocker.MagicMock(GET=GET) + assert get_redirect_url(mock_request, param_names) == expected_redirect @pytest.mark.parametrize( @@ -120,23 +140,41 @@ def test_custom_logout_view(mocker, client, user, is_authenticated, has_next): assert resp.url == (next_url if has_next else "/app") -def test_custom_login_view_authenticated_user_with_onboarding(mocker): - """Test CustomLoginView for an authenticated user who has never logged in before""" +@pytest.mark.parametrize( + ( + "req_data", + "expected_redirect", + ), + [ + ( + {"next": "/irrelevant", "signup_next": "/this?after=signup"}, + "/this?after=signup", + ), + ( + {"next": "/redirect?here=ok"}, # falls back to next + "/redirect?here=ok", + ), + ], +) +@pytest.mark.parametrize( + ("skip_onboarding", "expect_onboarding"), + [ + (None, True), # default behavior is to do onboarding + ("0", True), # explicit skip_onboarding=0 means do onboarding + ("1", False), # explicit skip_onboarding=1 means skip onboarding + ], +) +def test_custom_login_view_authenticated_user_needs_onboarding( + mocker, req_data, expected_redirect, skip_onboarding, expect_onboarding +): + """Test CustomLoginView for an authenticated user with incomplete onboarding""" factory = RequestFactory() - request = factory.get("/login/", {"next": "/dashboard"}) + if skip_onboarding is not None: + req_data["skip_onboarding"] = skip_onboarding + request = factory.get(reverse("login"), req_data) + request.user = MagicMock(is_anonymous=False) request.user.profile = MagicMock(has_logged_in=False) - - # Mock redirect to avoid URL resolution issues - mock_redirect = mocker.patch("authentication.views.redirect") - mock_redirect.return_value = MagicMock( - status_code=302, url="/onboarding?next=/search?resource=184" - ) - - mocker.patch("authentication.views.get_redirect_url", return_value="/dashboard") - mocker.patch( - "authentication.views.urlencode", return_value="next=/search?resource=184" - ) mocker.patch( "authentication.views.settings.MITOL_NEW_USER_LOGIN_URL", "/onboarding" ) @@ -145,72 +183,41 @@ def test_custom_login_view_authenticated_user_with_onboarding(mocker): response = CustomLoginView().get(request) assert response.status_code == 302 - assert response.url == "/onboarding?next=/search?resource=184" - - # Verify redirect was called with the onboarding URL - mock_redirect.assert_called_once_with("/onboarding?next=/search?resource=184") - -def test_custom_login_view_authenticated_user_skip_onboarding(mocker): - """Test skip_onboarding flag skips redirect to onboarding""" - factory = RequestFactory() - request = factory.get("/login/", {"next": "/dashboard", "skip_onboarding": "1"}) - request.user = MagicMock(is_anonymous=False) - request.user.profile = MagicMock(has_logged_in=False) - - # Mock redirect to avoid URL resolution issues - mock_redirect = mocker.patch("authentication.views.redirect") - mock_redirect.return_value = MagicMock(status_code=302, url="/dashboard") - - mocker.patch("authentication.views.get_redirect_url", return_value="/dashboard") - mocker.patch("authentication.views.decode_apisix_headers", return_value={}) - - response = CustomLoginView().get(request) - - # Verify has_logged_in was set to True and profile was saved - assert request.user.profile.has_logged_in is True - request.user.profile.save.assert_called_once() - - assert response.status_code == 302 - assert response.url == "/dashboard" + if expect_onboarding: + assert response.url == f"/onboarding?{urlencode({'next': expected_redirect})}" + else: + assert response.url == expected_redirect def test_custom_login_view_authenticated_user_who_has_logged_in_before(mocker): """Test that user who has logged in before is redirected to next url""" factory = RequestFactory() - request = factory.get("/login/", {"next": "/dashboard"}) + request = factory.get( + reverse("login"), + {"next": "/should-be-redirect?foo", "signup_next": "/irrelevant"}, + ) request.user = MagicMock(is_anonymous=False) request.user.profile = MagicMock(has_logged_in=True) - # Mock redirect to avoid URL resolution issues - mock_redirect = mocker.patch("authentication.views.redirect") - mock_redirect.return_value = MagicMock(status_code=302, url="/dashboard") - - mocker.patch("authentication.views.get_redirect_url", return_value="/dashboard") - mocker.patch("authentication.views.decode_apisix_headers", return_value={}) - response = CustomLoginView().get(request) assert response.status_code == 302 - assert response.url == "/dashboard" + assert response.url == "/should-be-redirect?foo" def test_custom_login_view_anonymous_user(mocker): """Test redirect for anonymous user""" factory = RequestFactory() - request = factory.get("/login/", {"next": "/dashboard"}) + request = factory.get( + reverse("login"), {"next": "/some-url", "signup_next": "/irrelevant"} + ) request.user = MagicMock(is_anonymous=True) - # Mock redirect to avoid URL resolution issues - mock_redirect = mocker.patch("authentication.views.redirect") - mock_redirect.return_value = MagicMock(status_code=302, url="/dashboard") - - mocker.patch("authentication.views.get_redirect_url", return_value="/dashboard") - response = CustomLoginView().get(request) assert response.status_code == 302 - assert response.url == "/dashboard" + assert response.url == "/some-url" def test_custom_login_view_first_time_login_sets_has_logged_in(mocker): @@ -228,12 +235,6 @@ def test_custom_login_view_first_time_login_sets_has_logged_in(mocker): request.user = mock_user - # Mock the redirect function to avoid URL resolution - mock_redirect = mocker.patch("authentication.views.redirect") - mock_redirect.return_value = MagicMock(status_code=302, url="/dashboard") - mocker.patch("authentication.views.get_redirect_url", return_value="/dashboard") - mocker.patch("authentication.views.decode_apisix_headers", return_value={}) - response = CustomLoginView().get(request) # Verify the response @@ -243,9 +244,6 @@ def test_custom_login_view_first_time_login_sets_has_logged_in(mocker): assert mock_profile.has_logged_in is True mock_profile.save.assert_called_once() - # Verify redirect was called with the correct URL - mock_redirect.assert_called_once_with("/dashboard") - @pytest.mark.parametrize( "test_case", diff --git a/docker-compose.apps.yml b/docker-compose.apps.yml index d847e01342..4e460607a2 100644 --- a/docker-compose.apps.yml +++ b/docker-compose.apps.yml @@ -59,7 +59,7 @@ services: command: > /bin/bash -c ' sleep 3; - celery -A main.celery:app worker -E -Q default,edx_content -B --scheduler redbeat.RedBeatScheduler -l ${MITOL_LOG_LEVEL:-INFO}' + celery -A main.celery:app worker -E -Q default,edx_content,embeddings -B --scheduler redbeat.RedBeatScheduler -l ${MITOL_LOG_LEVEL:-INFO}' depends_on: db: condition: service_healthy diff --git a/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.test.tsx b/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.test.tsx index a7cfbc1d77..184c891207 100644 --- a/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.test.tsx +++ b/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.test.tsx @@ -9,9 +9,10 @@ import { import * as commonUrls from "@/common/urls" import { Permission } from "api/hooks/user" import B2BAttachPage from "./B2BAttachPage" +import invariant from "tiny-invariant" // Mock next-nprogress-bar for App Router -const mockPush = jest.fn() +const mockPush = jest.fn() jest.mock("next-nprogress-bar", () => ({ useRouter: () => ({ push: mockPush, @@ -37,10 +38,18 @@ describe("B2BAttachPage", () => { }) await waitFor(() => { - expect(mockPush).toHaveBeenCalledWith( - expect.stringMatching(/login.*next=.*skip_onboarding=1/), - ) + expect(mockPush).toHaveBeenCalledOnce() }) + + const url = new URL(mockPush.mock.calls[0][0]) + expect(url.searchParams.get("skip_onboarding")).toBe("1") + const nextUrl = url.searchParams.get("next") + const signupNextUrl = url.searchParams.get("signup_next") + invariant(nextUrl) + invariant(signupNextUrl) + const attachView = commonUrls.b2bAttachView("test-code") + expect(new URL(nextUrl).pathname).toBe(attachView) + expect(new URL(signupNextUrl).pathname).toBe(attachView) }) test("Renders when logged in", async () => { diff --git a/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.tsx b/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.tsx index 9b1dcb66e3..195bb5d7b7 100644 --- a/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.tsx +++ b/frontends/main/src/app-pages/B2BAttachPage/B2BAttachPage.tsx @@ -42,9 +42,16 @@ const B2BAttachPage: React.FC = ({ code }) => { return } if (!user?.is_authenticated) { - const loginUrlString = urls.login({ - pathname: urls.b2bAttachView(code), - searchParams: new URLSearchParams(), + const loginUrlString = urls.auth({ + loginNext: { + pathname: urls.b2bAttachView(code), + searchParams: null, + }, + // On signup, redirect to the attach page so attachment can occur. + signupNext: { + pathname: urls.b2bAttachView(code), + searchParams: null, + }, }) const loginUrl = new URL(loginUrlString) loginUrl.searchParams.set("skip_onboarding", "1") diff --git a/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.test.tsx b/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.test.tsx index 4e6f817b5f..252a9d1ca9 100644 --- a/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.test.tsx +++ b/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.test.tsx @@ -1,6 +1,6 @@ import React from "react" import { renderWithProviders, screen, waitFor } from "../../test-utils" -import { HOME, login } from "@/common/urls" +import * as routes from "@/common/urls" import ForbiddenPage from "./ForbiddenPage" import { setMockResponse, urls, factories } from "api/test-utils" import { useUserMe } from "api/hooks/user" @@ -22,7 +22,7 @@ test("The ForbiddenPage loads with a link that directs to HomePage", async () => setMockResponse.get(urls.userMe.get(), makeUser({ is_authenticated: true })) renderWithProviders() const homeLink = await screen.findByRole("link", { name: "Home" }) - expect(homeLink).toHaveAttribute("href", HOME) + expect(homeLink).toHaveAttribute("href", routes.HOME) }) test("Fetches auth data afresh and redirects unauthenticated users to auth", async () => { @@ -53,9 +53,11 @@ test("Fetches auth data afresh and redirects unauthenticated users to auth", asy await waitFor(() => { expect(mockedRedirect).toHaveBeenCalledWith( - login({ - pathname: "/foo", - searchParams: new URLSearchParams({ cat: "meow" }), + routes.auth({ + loginNext: { + pathname: "/foo", + searchParams: new URLSearchParams({ cat: "meow" }), + }, }), ) }) diff --git a/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.tsx b/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.tsx index 1e840c5a06..25450f7747 100644 --- a/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.tsx +++ b/frontends/main/src/app-pages/ErrorPage/ForbiddenPage.tsx @@ -2,7 +2,7 @@ import React, { useEffect } from "react" import ErrorPageTemplate from "./ErrorPageTemplate" import { userQueries } from "api/hooks/user" import { useQuery } from "@tanstack/react-query" -import { redirectLoginToCurrent } from "@/common/client-utils" +import { redirectAuthToCurrent } from "@/common/client-utils" const ForbiddenPage: React.FC = () => { const user = useQuery({ @@ -15,7 +15,7 @@ const ForbiddenPage: React.FC = () => { useEffect(() => { if (shouldRedirect) { - redirectLoginToCurrent() + redirectAuthToCurrent() } }, [shouldRedirect]) diff --git a/frontends/main/src/app-pages/HomePage/HomePage.test.tsx b/frontends/main/src/app-pages/HomePage/HomePage.test.tsx index 792ed47070..27f1ab5897 100644 --- a/frontends/main/src/app-pages/HomePage/HomePage.test.tsx +++ b/frontends/main/src/app-pages/HomePage/HomePage.test.tsx @@ -295,9 +295,11 @@ describe("Home Page personalize section", () => { const link = within(personalize).getByRole("link") expect(link).toHaveAttribute( "href", - routes.login({ - pathname: routes.DASHBOARD_HOME, - searchParams: null, + routes.auth({ + loginNext: { + pathname: routes.DASHBOARD_HOME, + searchParams: null, + }, }), ) }) diff --git a/frontends/main/src/app-pages/HomePage/PersonalizeSection.tsx b/frontends/main/src/app-pages/HomePage/PersonalizeSection.tsx index ed1238afb9..e1d2167ae6 100644 --- a/frontends/main/src/app-pages/HomePage/PersonalizeSection.tsx +++ b/frontends/main/src/app-pages/HomePage/PersonalizeSection.tsx @@ -76,7 +76,9 @@ const AUTH_TEXT_DATA = { text: "As a member, get personalized recommendations, curate learning lists, and follow your areas of interest.", linkProps: { children: "Sign Up for Free", - href: urls.login({ pathname: urls.DASHBOARD_HOME, searchParams: null }), + href: urls.auth({ + loginNext: { pathname: urls.DASHBOARD_HOME, searchParams: null }, + }), }, }, } diff --git a/frontends/main/src/common/client-utils.ts b/frontends/main/src/common/client-utils.ts index 8480e6aa56..4003ef0feb 100644 --- a/frontends/main/src/common/client-utils.ts +++ b/frontends/main/src/common/client-utils.ts @@ -1,6 +1,5 @@ -"use client" import { ChannelCounts } from "api/v0" -import { login } from "./urls" +import { auth } from "./urls" import { redirect, usePathname, useSearchParams } from "next/navigation" const getSearchParamMap = (urlParams: URLSearchParams) => { @@ -52,7 +51,13 @@ const getCsrfToken = () => { ) } -const useLoginToCurrent = () => { +/** + * Returns a URL to authentication that redirects to current page after auth. + * + * By default, new users will be redirected to default signup destination. + * If `signup` is true, they will be redirected to the current page. + */ +const useAuthToCurrent = ({ signup }: { signup?: boolean } = {}) => { /** * NOTES: * 1. This is reactive; if current URL changes, the result of this hook @@ -64,21 +69,39 @@ const useLoginToCurrent = () => { */ const pathname = usePathname() const searchParams = useSearchParams() - return login({ pathname, searchParams }) + const current = { pathname, searchParams } + return auth({ + loginNext: current, + signupNext: signup ? current : undefined, + }) } /** - * Redirect user to login route with ?next=. + * Redirect user to auth. After auth: + * - existing users redirected to their current URL + * - new users redirected to the default signup destination, unless `signup` is true. */ -const redirectLoginToCurrent = (): never => { +const redirectAuthToCurrent = ({ + signup, +}: { + /** + * Whether to redirect signup to current; by default, false. New users + * will be redirected to the default signup destination. + */ + signup?: boolean +} = {}): never => { + const current = { + pathname: window.location.pathname, + searchParams: new URLSearchParams(window.location.search), + } redirect( /** * Calculating the ?next= via window.location is appropriate * here since it happens time of redirect call. */ - login({ - pathname: window.location.pathname, - searchParams: new URLSearchParams(window.location.search), + auth({ + loginNext: current, + signupNext: signup ? current : undefined, }), ) } @@ -88,6 +111,6 @@ export { aggregateProgramCounts, aggregateCourseCounts, getCsrfToken, - useLoginToCurrent, - redirectLoginToCurrent, + useAuthToCurrent, + redirectAuthToCurrent, } diff --git a/frontends/main/src/common/urls.test.ts b/frontends/main/src/common/urls.test.ts index 3a762250ee..2f35dbdb49 100644 --- a/frontends/main/src/common/urls.test.ts +++ b/frontends/main/src/common/urls.test.ts @@ -1,27 +1,35 @@ -import { login } from "./urls" +import { auth } from "./urls" const MITOL_API_BASE_URL = process.env.NEXT_PUBLIC_MITOL_API_BASE_URL test("login encodes the next parameter appropriately", () => { - expect(login({ pathname: null, searchParams: null })).toBe( - `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/`, + expect( + auth({ + loginNext: { pathname: null, searchParams: null }, + }), + ).toBe( + `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/&signup_next=http://test.learn.odl.local:8062/dashboard`, ) expect( - login({ - pathname: "/foo/bar", - searchParams: null, + auth({ + loginNext: { + pathname: "/foo/bar", + searchParams: null, + }, }), ).toBe( - `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/foo/bar`, + `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/foo/bar&signup_next=http://test.learn.odl.local:8062/dashboard`, ) expect( - login({ - pathname: "/foo/bar", - searchParams: new URLSearchParams("?cat=meow"), + auth({ + loginNext: { + pathname: "/foo/bar", + searchParams: new URLSearchParams("?cat=meow"), + }, }), ).toBe( - `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/foo/bar%3Fcat%3Dmeow`, + `${MITOL_API_BASE_URL}/login?next=http://test.learn.odl.local:8062/foo/bar%3Fcat%3Dmeow&signup_next=http://test.learn.odl.local:8062/dashboard`, ) }) diff --git a/frontends/main/src/common/urls.ts b/frontends/main/src/common/urls.ts index 14392545b3..cf8890a25e 100644 --- a/frontends/main/src/common/urls.ts +++ b/frontends/main/src/common/urls.ts @@ -55,40 +55,6 @@ if (process.env.NODE_ENV !== "production") { const MITOL_API_BASE_URL = process.env.NEXT_PUBLIC_MITOL_API_BASE_URL -export const LOGIN = `${MITOL_API_BASE_URL}/login` -export const LOGOUT = `${MITOL_API_BASE_URL}/logout/` - -/** - * Returns the URL to the login page, with a `next` parameter to redirect back - * to the given pathname + search parameters. - * - * NOTES: - * 1. useLoginToCurrent() is a convenience function that uses the current - * pathname and search parameters to generate the next URL. - * 2. `next` is required to encourage its use. You can explicitly pass `null` - * for values to skip them if desired. - */ -export const login = (next: { - pathname: string | null - searchParams: URLSearchParams | null - hash?: string | null -}) => { - const pathname = next.pathname ?? "/" - const searchParams = next.searchParams ?? new URLSearchParams() - const hash = next.hash ?? "" - /** - * To include search parameters in the next URL, we need to encode them. - * If we pass `?next=/foo/bar?cat=meow` directly, Django receives two separate - * parameters: `next` and `cat`. - * - * There's no need to encode the path parameter (it might contain slashes, - * but those are allowed in search parameters) so let's keep it readable. - */ - const search = searchParams?.toString() ? `?${searchParams.toString()}` : "" - const nextHref = `${ORIGIN}${pathname}${encodeURIComponent(search)}${encodeURIComponent(hash as string)}` - return `${LOGIN}?next=${nextHref}` -} - export const DASHBOARD_VIEW = "/dashboard/[tab]" const dashboardView = (tab: string) => generatePath(DASHBOARD_VIEW, { tab }) @@ -166,6 +132,61 @@ export const SEARCH_LEARNING_MATERIAL = querifiedSearchUrl({ resource_category: "learning_material", }) +export const LOGIN = `${MITOL_API_BASE_URL}/login` +export const LOGOUT = `${MITOL_API_BASE_URL}/logout/` + +type UrlDescriptor = { + pathname: string | null + searchParams: URLSearchParams | null + hash?: string | null +} +export type LoginUrlOpts = { + /** + * URL to redirect to after login. + */ + loginNext: UrlDescriptor + /** + * URL to redirect to after signup. + */ + signupNext?: UrlDescriptor +} + +const DEFAULT_SIGNUP_NEXT: UrlDescriptor = { + pathname: DASHBOARD_HOME, + searchParams: null, +} + +/** + * Returns the URL to the authentication page (login and signup). + * + * NOTES: + * 1. useAuthToCurrent() is a convenience function that uses the current + * pathname and search parameters to generate the next URL. + * 2. `next` is required to encourage its use. You can explicitly pass `null` + * for values to skip them if desired. + */ +export const auth = (opts: LoginUrlOpts) => { + const { loginNext, signupNext = DEFAULT_SIGNUP_NEXT } = opts + const encode = (value: UrlDescriptor) => { + const pathname = value.pathname ?? "/" + const searchParams = value.searchParams ?? new URLSearchParams() + const hash = value.hash ?? "" + /** + * To include search parameters in the next URL, we need to encode them. + * If we pass `?next=/foo/bar?cat=meow` directly, Django receives two separate + * parameters: `next` and `cat`. + * + * There's no need to encode the path parameter (it might contain slashes, + * but those are allowed in search parameters) so let's keep it readable. + */ + const search = searchParams?.toString() ? `?${searchParams.toString()}` : "" + return `${ORIGIN}${pathname}${encodeURIComponent(search)}${encodeURIComponent(hash)}` + } + const loginNextHref = encode(loginNext) + const signupNextHref = encode(signupNext) + return `${LOGIN}?next=${loginNextHref}&signup_next=${signupNextHref}` +} + export const ECOMMERCE_CART = "/cart/" as const export const B2B_ATTACH_VIEW = "/attach/[code]" diff --git a/frontends/main/src/components/RestrictedRoute/RestrictedRoute.tsx b/frontends/main/src/components/RestrictedRoute/RestrictedRoute.tsx index 52ab28baf1..1daa5e8242 100644 --- a/frontends/main/src/components/RestrictedRoute/RestrictedRoute.tsx +++ b/frontends/main/src/components/RestrictedRoute/RestrictedRoute.tsx @@ -3,7 +3,7 @@ import React, { useEffect } from "react" import { ForbiddenError } from "@/common/errors" import { Permission, userQueries } from "api/hooks/user" -import { redirectLoginToCurrent } from "@/common/client-utils" +import { redirectAuthToCurrent } from "@/common/client-utils" import { useQuery } from "@tanstack/react-query" type RestrictedRouteProps = { @@ -54,7 +54,7 @@ const RestrictedRoute: React.FC = ({ * and any "secret" data is gated via API auth checks anyway. */ if (shouldRedirect) { - redirectLoginToCurrent() + redirectAuthToCurrent() } }, [shouldRedirect]) if (isLoading) return null diff --git a/frontends/main/src/page-components/Header/Header.test.tsx b/frontends/main/src/page-components/Header/Header.test.tsx index 158cc55491..379f7e49c5 100644 --- a/frontends/main/src/page-components/Header/Header.test.tsx +++ b/frontends/main/src/page-components/Header/Header.test.tsx @@ -60,9 +60,11 @@ describe("UserMenu", () => { test("Unauthenticated users see the Sign Up / Login link", async () => { const isAuthenticated = false const initialUrl = "/foo/bar?cat=meow" - const expectedUrl = urlConstants.login({ - pathname: "/foo/bar", - searchParams: new URLSearchParams("?cat=meow"), + const expectedUrl = urlConstants.auth({ + loginNext: { + pathname: "/foo/bar", + searchParams: new URLSearchParams("?cat=meow"), + }, }) setMockResponse.get(urls.userMe.get(), { is_authenticated: isAuthenticated, diff --git a/frontends/main/src/page-components/Header/UserMenu.tsx b/frontends/main/src/page-components/Header/UserMenu.tsx index 6619b858bc..11e06b4272 100644 --- a/frontends/main/src/page-components/Header/UserMenu.tsx +++ b/frontends/main/src/page-components/Header/UserMenu.tsx @@ -12,7 +12,7 @@ import { } from "@remixicon/react" import { useUserMe, User } from "api/hooks/user" import MITLogoLink from "@/components/MITLogoLink/MITLogoLink" -import { useLoginToCurrent } from "@/common/client-utils" +import { useAuthToCurrent } from "@/common/client-utils" const FlexContainer = styled.div({ display: "flex", @@ -125,7 +125,7 @@ type UserMenuProps = { const UserMenu: React.FC = ({ variant }) => { const [visible, setVisible] = useState(false) - const loginUrl = useLoginToCurrent() + const loginUrl = useAuthToCurrent() const { isLoading, data: user } = useUserMe() if (isLoading) { diff --git a/frontends/main/src/page-components/SignupPopover/SignupPopover.test.tsx b/frontends/main/src/page-components/SignupPopover/SignupPopover.test.tsx index 98f8917064..0e9b9cac6f 100644 --- a/frontends/main/src/page-components/SignupPopover/SignupPopover.test.tsx +++ b/frontends/main/src/page-components/SignupPopover/SignupPopover.test.tsx @@ -15,9 +15,15 @@ test("SignupPopover shows link to sign up", async () => { const link = within(dialog).getByRole("link") invariant(link instanceof HTMLAnchorElement) expect(link.href).toMatch( - urls.login({ - pathname: "/some-path", - searchParams: new URLSearchParams("dog=woof"), + urls.auth({ + loginNext: { + pathname: "/some-path", + searchParams: new URLSearchParams("dog=woof"), + }, + signupNext: { + pathname: "/some-path", + searchParams: new URLSearchParams("dog=woof"), + }, }), ) }) diff --git a/frontends/main/src/page-components/SignupPopover/SignupPopover.tsx b/frontends/main/src/page-components/SignupPopover/SignupPopover.tsx index 6395aae5dc..d70fb5bb89 100644 --- a/frontends/main/src/page-components/SignupPopover/SignupPopover.tsx +++ b/frontends/main/src/page-components/SignupPopover/SignupPopover.tsx @@ -2,7 +2,8 @@ import React from "react" import { Popover, Typography, styled } from "ol-components" import { ButtonLink } from "@mitodl/smoot-design" import type { PopoverProps } from "ol-components" -import { useLoginToCurrent } from "@/common/client-utils" + +import { useAuthToCurrent } from "@/common/client-utils" const StyledPopover = styled(Popover)({ width: "300px", @@ -31,7 +32,7 @@ type SignupPopoverProps = Pick< "anchorEl" | "onClose" | "placement" > const SignupPopover: React.FC = (props) => { - const loginUrl = useLoginToCurrent() + const loginUrl = useAuthToCurrent({ signup: true }) return ( diff --git a/learning_resources/constants.py b/learning_resources/constants.py index ef2c835fdd..58e52673d4 100644 --- a/learning_resources/constants.py +++ b/learning_resources/constants.py @@ -181,7 +181,9 @@ class LearningResourceRelationTypes(TextChoices): zip(VALID_COURSE_CONTENT_TYPES, VALID_COURSE_CONTENT_TYPES) ) -VALID_TUTOR_PROBLEM_TYPES = ["problem", "solution"] +TUTOR_PROBLEM_TYPE = "problem" +TUTOR_SOLUTION_TYPE = "solution" +VALID_TUTOR_PROBLEM_TYPES = [TUTOR_PROBLEM_TYPE, TUTOR_SOLUTION_TYPE] VALID_TUTOR_PROBLEM_TYPE_CHOICES = list( zip(VALID_TUTOR_PROBLEM_TYPES, VALID_TUTOR_PROBLEM_TYPES) ) diff --git a/learning_resources/etl/canvas.py b/learning_resources/etl/canvas.py index 11534fc0aa..b3dc5f8b82 100644 --- a/learning_resources/etl/canvas.py +++ b/learning_resources/etl/canvas.py @@ -21,7 +21,8 @@ from PIL import Image from learning_resources.constants import ( - VALID_TUTOR_PROBLEM_TYPES, + TUTOR_PROBLEM_TYPE, + TUTOR_SOLUTION_TYPE, LearningResourceType, PlatformType, ) @@ -298,11 +299,24 @@ def transform_canvas_problem_files( path = file_data["source_path"] path = path[len(settings.CANVAS_TUTORBOT_FOLDER) :] path_parts = path.split("/", 1) + + if len(path_parts) != 2: # noqa: PLR2004 + log.warning( + "unnested file in problem folder for course run %s: %s", + run.id, + path, + ) + continue + problem_file_data["problem_title"] = path_parts[0] - for problem_type in VALID_TUTOR_PROBLEM_TYPES: - if problem_type in path_parts[1].lower(): - problem_file_data["type"] = problem_type - break + + problem_file_data["file_name"] = path_parts[1] + + if TUTOR_SOLUTION_TYPE in problem_file_data["file_name"].lower(): + problem_file_data["type"] = TUTOR_SOLUTION_TYPE + else: + problem_file_data["type"] = TUTOR_PROBLEM_TYPE + if ( problem_file_data["file_extension"].lower() == ".pdf" and settings.CANVAS_PDF_TRANSCRIPTION_MODEL diff --git a/learning_resources/factories.py b/learning_resources/factories.py index fc3f81fe26..b7131329de 100644 --- a/learning_resources/factories.py +++ b/learning_resources/factories.py @@ -886,6 +886,7 @@ class TutorProblemFileFactory(DjangoModelFactory): type = FuzzyChoice("problem", "solution") content = factory.Faker("text") source_path = factory.Faker("file_path", extension="txt") + file_name = factory.LazyAttribute(lambda o: o.source_path.split("/")[-1]) @classmethod def _create(cls, model_class, *args, **kwargs): diff --git a/learning_resources/migrations/0096_tutorproblemfile_file_name.py b/learning_resources/migrations/0096_tutorproblemfile_file_name.py new file mode 100644 index 0000000000..1c46ee9c68 --- /dev/null +++ b/learning_resources/migrations/0096_tutorproblemfile_file_name.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.24 on 2025-09-22 19:45 + +from django.db import migrations, models + + +def populate_file_name(apps, schema_editor): + for tutor_problem_file in apps.get_model( + "learning_resources", "TutorProblemFile" + ).objects.all(): + if tutor_problem_file.file_name is None: + tutor_problem_file.file_name = tutor_problem_file.source_path.split("/")[-1] + tutor_problem_file.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0095_learningresource_require_summaries"), + ] + + operations = [ + migrations.AddField( + model_name="tutorproblemfile", + name="file_name", + field=models.CharField(blank=True, max_length=256, null=True), + ), + migrations.RunPython(populate_file_name, migrations.RunPython.noop), + ] diff --git a/learning_resources/models.py b/learning_resources/models.py index 400a2f98b0..1d897e8c56 100644 --- a/learning_resources/models.py +++ b/learning_resources/models.py @@ -918,6 +918,7 @@ class TutorProblemFile(TimestampedModel): archive_checksum = models.CharField(max_length=32, null=True, blank=True) # noqa: DJ001 source_path = models.CharField(max_length=1024, null=True, blank=True) # noqa: DJ001 file_extension = models.CharField(max_length=32, null=True, blank=True) # noqa: DJ001 + file_name = models.CharField(max_length=256, null=True, blank=True) # noqa: DJ001 class ContentFile(TimestampedModel): diff --git a/learning_resources/views.py b/learning_resources/views.py index 2c032cb4e4..79e6b17bf1 100644 --- a/learning_resources/views.py +++ b/learning_resources/views.py @@ -1439,13 +1439,32 @@ def retrieve_problem(self, request, run_readable_id, problem_title): # noqa: AR run_id=run_readable_id, learning_resource__platform=PlatformType.canvas.name ).first() - problem_file = TutorProblemFile.objects.get( + problem_files = TutorProblemFile.objects.filter( run=run, problem_title=problem_title, type="problem" ) - solution_file = TutorProblemFile.objects.get( + solution_files = TutorProblemFile.objects.filter( run=run, problem_title=problem_title, type="solution" ) return Response( - {"problem_set": problem_file.content, "solution_set": solution_file.content} + { + # problem_set and solution_set will be removed after i make the + # required changes to open_learning_ai_tutor + "problem_set": problem_files.first().content, + "solution_set": solution_files.first().content, + "problem_set_files": [ + { + "file_name": problem_file.file_name, + "content": problem_file.content, + } + for problem_file in problem_files + ], + "solution_set_files": [ + { + "file_name": solution_file.file_name, + "content": solution_file.content, + } + for solution_file in solution_files + ], + } ) diff --git a/learning_resources/views_test.py b/learning_resources/views_test.py index 23d0960ad4..61dac87d01 100644 --- a/learning_resources/views_test.py +++ b/learning_resources/views_test.py @@ -1455,6 +1455,15 @@ def test_course_run_problems_endpoint(client, user_role, django_user_model): problem_title="Problem Set 1", type="problem", content="Content for Problem Set 1", + file_name="problem1.txt", + ) + + TutorProblemFileFactory.create( + run=course_run, + problem_title="Problem Set 1", + type="problem", + content="Content for Problem Set 1 Part 2", + file_name="problem1-b.txt", ) TutorProblemFileFactory.create( @@ -1462,6 +1471,7 @@ def test_course_run_problems_endpoint(client, user_role, django_user_model): problem_title="Problem Set 1", type="solution", content="Content for Problem Set 1 Solution", + file_name="solution1.txt", ) TutorProblemFileFactory.create( run=course_run, problem_title="Problem Set 2", type="problem" @@ -1487,6 +1497,19 @@ def test_course_run_problems_endpoint(client, user_role, django_user_model): assert detail_resp.json() == { "problem_set": "Content for Problem Set 1", "solution_set": "Content for Problem Set 1 Solution", + "problem_set_files": [ + {"file_name": "problem1.txt", "content": "Content for Problem Set 1"}, + { + "file_name": "problem1-b.txt", + "content": "Content for Problem Set 1 Part 2", + }, + ], + "solution_set_files": [ + { + "file_name": "solution1.txt", + "content": "Content for Problem Set 1 Solution", + }, + ], } elif user_role == "normal": assert detail_resp.status_code == 403 diff --git a/main/celery.py b/main/celery.py index be308be6fa..f16e51c1e0 100644 --- a/main/celery.py +++ b/main/celery.py @@ -17,6 +17,7 @@ app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) # pragma: no cover app.conf.task_routes = { + "vector_search.tasks.generate_embeddings": {"queue": "embeddings"}, "learning_resources.tasks.get_content_tasks": {"queue": "edx_content"}, "learning_resources.tasks.get_content_files": {"queue": "edx_content"}, "learning_resources.tasks.import_all_xpro_files": {"queue": "edx_content"}, diff --git a/main/settings.py b/main/settings.py index 5490024a72..8d577c6d2f 100644 --- a/main/settings.py +++ b/main/settings.py @@ -34,7 +34,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.45.0" +VERSION = "0.45.1" log = logging.getLogger() diff --git a/main/settings_celery.py b/main/settings_celery.py index eb6c0e265f..a8ff528b38 100644 --- a/main/settings_celery.py +++ b/main/settings_celery.py @@ -60,9 +60,7 @@ }, "update-podcasts": { "task": "learning_resources.tasks.get_podcast_data", - "schedule": get_int( - "PODCAST_FETCH_SCHEDULE_SECONDS", 60 * 60 * 2 - ), # default is every 2 hours + "schedule": crontab(minute=0, hour="6,23"), # 2am and 7pm EST }, "update-professional-ed-resources-every-1-days": { "task": "learning_resources.tasks.get_mitpe_data", @@ -80,9 +78,7 @@ }, "update-youtube-videos": { "task": "learning_resources.tasks.get_youtube_data", - "schedule": get_int( - "YOUTUBE_FETCH_SCHEDULE_SECONDS", 60 * 30 - ), # default is every 30 minutes + "schedule": crontab(minute=30, hour=8), # 4:30am EST }, "update-youtube-transcripts": { "task": "learning_resources.tasks.get_youtube_transcripts", diff --git a/poetry.lock b/poetry.lock index de68bb9b96..67c0936349 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2432,12 +2432,6 @@ files = [ {file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:447fc2d49a41449684154c12c03ab80176a413e9810d974363a061b71bdbf5a0"}, {file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4598c2aa14c866a10a07a2944e2c212f53d0c337ce211336ad68ae8243646216"}, {file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:69d2bd7ab7f94a6c73325f4b88fd07b0d5f4865672ed7a519f2d896949353761"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:45a3f7e3531dd2650f5bb840ed11ce77d0eeb45d0f4c9cd6985eb805e17490e6"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:73b427e0ea8c2750ee05980196893287bfc9f2a155a282c0f248b472ea7ae3e7"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2959ef84271e4fa646c3dbaad9e6f2912bf54dcdfefa5999c2ef7c927d92127"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a800fcb8e53a8f4a7c02b4b403d2325a16cad63a877e57bd603aa50bf0e475b"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:528321e9aab686435ba09cc6ff90f12e577ace79762f74831ec2265eeab624a8"}, - {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:034be44ff3318359e3c678cb5c4ed13efd69aeb558f2981a32bd3e3fb5355700"}, {file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a3182f1457599c2901c48a1def37a5bc4762f696077e186e2050fcc60b2fbdf"}, {file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:86b489238dc2cbfa53cdd5621e888786a53031d327e0a8509529c7568292b0ce"}, {file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4c8aca6ab5da4211870c1d8410c699a9d543e86304aac47e1558ec94d0da97a"}, @@ -2563,6 +2557,128 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4 [package.extras] grpc = ["grpcio (>=1.44.0,<2.0.0)"] +[[package]] +name = "granian" +version = "2.5.4" +description = "A Rust HTTP server for Python applications" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "granian-2.5.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:907d17f94a039b1047a82386b4979a6a7db7f4c37598225c6184a2b89f0ae12d"}, + {file = "granian-2.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a009e99d3c4a2a70a15a97391566753045a81641e5a3e651ff346d8bb7fe7450"}, + {file = "granian-2.5.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cb602ac3ea567476c339e8683a0fa2ffe7fd8432798bd63c371d5b32502bdb9"}, + {file = "granian-2.5.4-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f52aee85459304f1e74ff4cb5bb60d23db267b671b1199ff589b1a5a65f5638f"}, + {file = "granian-2.5.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8478d777971d6c1601b479c70a5c1aaaba7b656fa5044b1c38b4ba5e172f0fc7"}, + {file = "granian-2.5.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1da588e951b3e0bce94f2743158750c9733efcbe5c27b31f50e9bda6af8aac1f"}, + {file = "granian-2.5.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:79db7d0eac7445a383e22b6d3e9323882bc9a9c1d2fd62097c0452822c4de526"}, + {file = "granian-2.5.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:cc75f15876415054c094e9ef941cf49c315ee5f0f20701fdfb3ffc698054c727"}, + {file = "granian-2.5.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2caeee9d12144c9c285d3074c7979cdf1ad3d84a86204dec9035ca6cec5d713f"}, + {file = "granian-2.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:9b1d2e2af1829e7022dd89bcbfda0e580a1bc1440105757f16ef262f8ab3aa5e"}, + {file = "granian-2.5.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a404bff75dc29c01566a4e896237f6cb8eda49a71b90770b8316ebe1b08a3d46"}, + {file = "granian-2.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d91b4642283ea8169aad64b74b242c364a3ce24d6aeed9b5a4358f99a5ab4d84"}, + {file = "granian-2.5.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6aa6b4ad4d479fe3e7d42ca4321ae7febad9cdae5c269032234b8b4ac8dbd017"}, + {file = "granian-2.5.4-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2466523c14724d2d68497cd081ffd2aa913381be199e7eb71347847a3651224c"}, + {file = "granian-2.5.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce9ec6baebb83ba7d1ed507dc7d301f7f29725f9b7a8c9c974f96479dea3a090"}, + {file = "granian-2.5.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:8b3faa2eec6dbbb072aae575d6a6a5e5577aef13c93d38d454a6a9fffc954ce7"}, + {file = "granian-2.5.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:25a1d03fc93184009d5e76a5bfb5b29222e7debacfc225dd5d3732f6f6f99c10"}, + {file = "granian-2.5.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:1e580f5fa30ed04e724c71de099dcacc4722613f8a53e41454bac86242887da7"}, + {file = "granian-2.5.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5a4e74bf6d91dd7df6ffc7edb74e74147057fc947c04684d2d9af03e5e71ad71"}, + {file = "granian-2.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:e28453632093da4e31b1d346396362d1f002ac1bf766486b4f8943362e081021"}, + {file = "granian-2.5.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:c4387cca4e38ec7579cac71a2da27fd177ced682b1de8bf917b9610f2ac0ba5e"}, + {file = "granian-2.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a126b75927583292a9d2cfae627cd4b1da3e68d04dd87ba5a53b4860768d9e04"}, + {file = "granian-2.5.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b44dc391bf9bc1303bcb2cb344bbb5c35de92f43a3e8584f2a984dfda2fea8e3"}, + {file = "granian-2.5.4-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07c47847163a1bcce0b7c323093b20be8a8ec9d4f4eba596b4d27f85ddbe669f"}, + {file = "granian-2.5.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6c50539f654ce5f8fadd9b360fac0361d812c39c7a5f1e525889c51899a10f0"}, + {file = "granian-2.5.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e52f65acd4da0a3af7a5d2d6a6445d530e84fe52057ee39f52ce92a6598fe37b"}, + {file = "granian-2.5.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b78ab23495e384521085c33fecb3987779e1b1e43f34acd5b25e864b699933f9"}, + {file = "granian-2.5.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:6a477b204fca30218b3cc16721df38f1e159c5ee27252b305c78982af1995974"}, + {file = "granian-2.5.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7f58116ab1300ca744a861482ce3194e9be5f1adad9ac4adda89d47b1ba3fa50"}, + {file = "granian-2.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:1fb3c2a76eff474b4a9611441fbf14da760d002a7bc6b37dee6f970dc642c862"}, + {file = "granian-2.5.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:533bf842b56c8531705048211e3152fb1234d7611f83257a71cbf7e734c0f4a1"}, + {file = "granian-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d1efb111f84236a72d5864d64a0198e04e699014119c33d957fac34a0efb2474"}, + {file = "granian-2.5.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0341a553fe913b4a741c10f532f5315d57deaa34877494d4c4b09c666f5266c"}, + {file = "granian-2.5.4-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b3b24b7620df45752bbf34f93250f733a6996a7409879efbea6ab38f57eff69"}, + {file = "granian-2.5.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bb902636271f9e61f6653625670982f7a0e19cbc7ae56fc65cd26bf330c612f"}, + {file = "granian-2.5.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:23b2e86ea97320bbe80866a94e6855752f0c73c0ec07772e0241e8409384cde5"}, + {file = "granian-2.5.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:328ed82315ccbd5dedc8f5318a1f80d49e07eb34ebc0753bc2930d2f30070a34"}, + {file = "granian-2.5.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:9bd438bb41cbac25f5f3155924947f0e2594b69f8a5f78e43c453c35fa28a1f0"}, + {file = "granian-2.5.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d6d1b462ccb57af3051def8eae13f1df81dc902e9deff3cc6dfbb692c40a5a1f"}, + {file = "granian-2.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:7fa841c10420b2c0a2001db6d75bcc8d171ed2e93edd00d8223b4e7cc0d47500"}, + {file = "granian-2.5.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:d04a1432ed98b7e4b4e5cff188819f34bd680785352237477d7886eb9305f692"}, + {file = "granian-2.5.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c6309d729f1b022b09fd051a277096a732bd8ed39986ac4b9849f6e79b660880"}, + {file = "granian-2.5.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a067c27e733b0851e6f66175c1aac8badda60b698457181881c08a4e17baecf"}, + {file = "granian-2.5.4-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:54bd0604db172a964b1bc4b8709329b7f4e9cff6b8f468104ca7603a5e61d529"}, + {file = "granian-2.5.4-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:487bdc40b234ef84510eac1d63f0720ca92daca08feb7d2d98a1f0a84cc54b0e"}, + {file = "granian-2.5.4-cp313-cp313t-musllinux_1_1_armv7l.whl", hash = "sha256:76cd55ab521cc501a977f50ace9d72a6a4f9a6849e6490b14af2e9acc614ce55"}, + {file = "granian-2.5.4-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:27b3e7906916ad6e84a8b16d89517d8652bece62888cb9421597eb14767a9d92"}, + {file = "granian-2.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:bb6be969da8b9c76274850214ba070dacd330ee61e5d9cc3c5390f865a0cf67c"}, + {file = "granian-2.5.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c237db56e5ff3fdad6539a3fbfcb9b57ce71463db55a016ba08296828043112f"}, + {file = "granian-2.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a76d7033a8d68c8353293fae8365e3b649bb155ab39af14387f3e9e870d503fb"}, + {file = "granian-2.5.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6778b9f7ecef8a423dd203aa5b0644a18d53eb749e830b2fe33abecad5d7e84"}, + {file = "granian-2.5.4-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3accc02c981436e783772b12ea8cead35e8e644437881d7da842ff474f8e30f9"}, + {file = "granian-2.5.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3eaaf38851308af616ad5fdc35f333857f128493072ea208c1bb2fb557dcf2e"}, + {file = "granian-2.5.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:3cad64e8e41d6f3daf3e7a3eea88023aa7e64ee81450443ac9f4e6cae005079d"}, + {file = "granian-2.5.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4bb60b122971326d0d1baf10336c67bdecdd7adc708cf0b09bf1cde5563e8f5"}, + {file = "granian-2.5.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:14129339f0ed9bbd2d129f82ed16e0c330edca7300482bd53cef466cc7b3ec6d"}, + {file = "granian-2.5.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:294c574fcd8005587671476b751e5853b262ecb1b1011e634ac160d6a0259abd"}, + {file = "granian-2.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:e8082ae43ac2f080071bdef557b205bcc4b2f71d8a148a33e389329935533e17"}, + {file = "granian-2.5.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3f0940963f6c6e50de19195014902d9a565a612aa0583082c9082bd83f259446"}, + {file = "granian-2.5.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bc27bff6ea5a80fd5bf28297ac53fa31771cbdfa35650a6eb4f2c4efc751097d"}, + {file = "granian-2.5.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73c136bac23cd1d2c969a52374972ec7da6e0920768bf0bcce65e00cabb4ebb9"}, + {file = "granian-2.5.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2dc03e2f375354b95d477c9455fb2fb427a922740f45e036cdf60da660adbf13"}, + {file = "granian-2.5.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:539ee12b02281929358349e01a0c42c0594ebcf4f44033c8a4d7a446f837e034"}, + {file = "granian-2.5.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:97735bdbc2877583ea1c8dbfca31bcaf118a6e818afe6000eb8a9d09fd9d07e0"}, + {file = "granian-2.5.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:5f642a4fa1d41943d288db714bd1e0d218537bfa8bc6355d7063e8959b84c32b"}, + {file = "granian-2.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:6635c18a4af7dea3c16f81143831ef540dd671829712fa1d8fcedd2da1cd8ae3"}, + {file = "granian-2.5.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:16bf4d4f78a9703c8ecd1484a5321c9edb001d9d7e59a0e7cf044462afc9cd77"}, + {file = "granian-2.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c7e6f92f87115c86162e294eb1bd809cca7fc68c75a1c1b3a4bd916568c74ee7"}, + {file = "granian-2.5.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:71378e1c0201a418fce6852c09d80de6ed0e6ff89dd019e4efb7ab42bff3461d"}, + {file = "granian-2.5.4-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e39bc221a08b3afdb6d5e53dadff929d76f3b725d9534a00acc7a1260099c727"}, + {file = "granian-2.5.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2197096e82f31388f329d6f0e4c13746a1e7a07db5331a77becb1ac6fb18fd"}, + {file = "granian-2.5.4-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab64839e7395b45fc9918eacc4da3768b02256a0eb231d9bc533ab83ab60af40"}, + {file = "granian-2.5.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9e3cc6fbfc0a8ed35e2237fe2aee706cc7ca16c59bf332b44508d1bffdc33c69"}, + {file = "granian-2.5.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:3923fb93dbf51e88492f61244c4ddc1980eaf9cb0c2ee212d83c3e6ffb97c3d9"}, + {file = "granian-2.5.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:89b884962aa03c98743304dd85f1ccff216cd4d8125889bdc2fe4260025c9db0"}, + {file = "granian-2.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:aec61eb6b2031860704c8c1e462148b3c59759a28e19df72702c49bf15caa276"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9b970a50230ae437615d754e1bc4aaa740fbe3f1418cc0c8933b260a691bb8f5"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a09d2bef7805f10093aa356d976fdb3607d274252ef9429c6c1a24d239032c29"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6d5bd05c6e833d54b77c2ee19130cfa5d54ae4eb301ffca56744f712c4a9d03"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7782a25ab78a55b61cb9b06f8aac471e9fafa3e1c20d6cdf970e93c834f6ddf"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4403600ac0273d4169c4c73773f57c5a3b44cc8aa8384a2f468c98c4294a3f27"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c711335619a6936728b7773052fb0ec9d612b19abb2c786972ce3efee836df9d"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ff9996929a16a72a61fb1f03a9e79e830bf7a6a4e9eb0470c6ef03f67d5ea5c0"}, + {file = "granian-2.5.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d069aba17c966a7809fe373b772685110e6e5e472f97ff9155dcd8e681027a9f"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3fcf2e6c8a150f48e4b77271b33ebfc7c2d8381e692b5177d4bd7fcefbb691d"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c5550d533a61053fc4d95044bdc80ba00118ca312ed85867ed63163fa5878f85"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48356114113ac3d48f70ea05cf42e560384c93318f5ef8f5638cb666f8243f2b"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:cbadc8e49f90716b8f8aa2c2cee7a2e82c5a37dab5f6fbd449e76185ce205715"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d1712a8ede152c2648557de6a23dbeb05ed499bfd83c42dad0689d9f2ba0621d"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:b78b8a6f30d0534b2db3f9cb99702d06be110b6e91f5639837a6f52f4891fc1d"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:10ae5ce26b1048888ac5aa3747415d8be8bbb014106b27ef0e77d23d2e00c51d"}, + {file = "granian-2.5.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:602fc22b2fde1c624cbc4ee64b0c8e547d394fbef70927ec7b0ca5e2d1ccea31"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b860e75c7755cfcf65c7f241e13750e78fb9b144d591ce0ec0dab02fae989fb5"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:29a0d2a5d2e77ae72292e56d7b66dbbabaf13ba3249e0d26c7a0e479384839ce"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09422fde80f418577cd16c7a2f4fb9c917f5282760694f36bb225c0864f139b5"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a9cd021fcece7572d16f0560b375539ff06b1ee468fc991ec6718925cd1f07e6"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0846be8ba1d8ac536607e0e5a4538bb4556eac322224a0d18edad2498f2ea318"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:cab6062978fd3494216c2e8bf1edd88444fe2ccd58fc6a5a39f899993001d44c"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ac3fe7dbef11059c24cf7461a88210b23e69ab34389e5c49387d926658bb53e7"}, + {file = "granian-2.5.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f73edfcc140259195c1ad8b556a986d3044e1225f50ca38aa8da7514fc7ea170"}, + {file = "granian-2.5.4.tar.gz", hash = "sha256:85989a08052f1bbb174fd73759e1ae505e50b4c0690af366ca6ba844203dd463"}, +] + +[package.dependencies] +click = ">=8.0.0" + +[package.extras] +all = ["granian[dotenv,pname,reload]"] +dotenv = ["python-dotenv (>=1.1,<2.0)"] +pname = ["setproctitle (>=1.3.3,<1.4.0)"] +reload = ["watchfiles (>=1.0,<2.0)"] +rloop = ["rloop (>=0.1,<1.0) ; sys_platform != \"win32\""] +uvloop = ["uvloop (>=0.18.0) ; platform_python_implementation == \"CPython\" and sys_platform != \"win32\""] + [[package]] name = "greenlet" version = "3.2.2" @@ -2625,7 +2741,6 @@ files = [ {file = "greenlet-3.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a8fa80665b1a29faf76800173ff5325095f3e66a78e62999929809907aca5659"}, {file = "greenlet-3.2.2-cp39-cp39-win32.whl", hash = "sha256:6629311595e3fe7304039c67f00d145cd1d38cf723bb5b99cc987b23c1433d61"}, {file = "greenlet-3.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834"}, - {file = "greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485"}, ] markers = {main = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} @@ -9119,4 +9234,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "~3.12" -content-hash = "29b3bcb4d3e513deec43a0228f2a29ac2390afa3c0b414bd993ba22bdd58c9c0" +content-hash = "d79dd5e782dbf31ad9f38ee63a42d2d624e499d46c081297a15dcddfb86c373e" diff --git a/pyproject.toml b/pyproject.toml index 794c5efa40..9ea4298904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ drf-nested-routers = "^0.94.0" drf-spectacular = "^0.28.0" feedparser = "^6.0.10" google-api-python-client = "^2.89.0" +granian = "^2.5.4" html2text = "^2025.0.0" html5lib = "^1.1" ipython = "^9.0.0" diff --git a/vector_search/tasks.py b/vector_search/tasks.py index 461a6692a1..d80bf91373 100644 --- a/vector_search/tasks.py +++ b/vector_search/tasks.py @@ -338,7 +338,9 @@ def embed_run_content_files(self, run_id): celery.group( [ generate_embeddings.si(ids, CONTENT_FILE_TYPE, overwrite=True) - for ids in chunks(content_file_ids) + for ids in chunks( + content_file_ids, chunk_size=settings.QDRANT_CHUNK_SIZE + ) ] ) ) @@ -355,6 +357,6 @@ def remove_run_content_files(run_id): return celery.group( [ remove_embeddings.si(ids, CONTENT_FILE_TYPE) - for ids in chunks(content_file_ids) + for ids in chunks(content_file_ids, chunk_size=settings.QDRANT_CHUNK_SIZE) ] )