Skip to content

Refine task progress #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
saimouu wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Refine task progress #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6a85b15
Return job status and task counts from /job?project=uuid
saimouu Feb 10, 2026
58c3ab3
Send job progress event from backend
saimouu Feb 10, 2026
cc8f748
Make job stats consistent and add JobModel to front
saimouu Feb 11, 2026
0c48812
Get job updates to front via event stream
saimouu Feb 12, 2026
97df1e2
Remove manual celery failure state update and move job progress event…
saimouu Feb 18, 2026
ac11232
Rename doneCount
saimouu Feb 18, 2026
7e93aa4
Merge branch 'main' into feature/refine-task-progress
saimouu Feb 18, 2026
a01deba
fix lint
saimouu Feb 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions client/src/components/EventStream.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { useCallback, useEffect, useState } from "react";
import { CircleAlert } from "lucide-react";
import * as z from "zod";
import classNames from "classnames";
import { useTypedStoreActions } from "../state/store";
import type { JobStats } from "../state/types";

const EventName = {
// Events for JobTask-related things
Expand All @@ -16,6 +18,7 @@ const EventName = {
// Events for Job-related things
JOB_COMPLETE: 3001,
JOB_CREATED: 3002,
JOB_PROGRESS: 3003,
// Events for Project-related things
PROJECT_CREATED: 4001,
PROJECT_FILE_UPLOADED: 4002,
Expand Down Expand Up @@ -84,19 +87,37 @@ export const EventStream = () => {
const [connected, setConnected] = useState(false);
const [logs, setLogs] = useState<Array<EventData>>([]);

// TODO: Integrate into state management to update job task status in real time
// const setJobsForProject = useTypedStoreActions(
// (actions) => actions.setJobsForProject
// );
const updateJobStats = useTypedStoreActions(
(actions) => actions.updateJobStats
);


const _onMessage = useCallback((event: MessageEvent<unknown>) => {
const { data } = event;
if (typeof data === "string") {
const dataJson = JSON.parse(data);
// console.log(dataJson);
const parsedData = EventData.safeParse(dataJson);
if (!parsedData.error) {
setLogs((logs) => [...logs, parsedData.data]);
const eventData = parsedData.data;
setLogs((logs) => [...logs, eventData]);

switch (eventData.event_name) {
case EventName.JOB_PROGRESS:
{
const jobId = eventData.value.job_id;
const stats: JobStats = eventData.value.stats;
updateJobStats({ jobId, stats });
}
break;
default:
break;
}
}
}
}, []);
}, [updateJobStats]);

const startLogStream = useCallback(() => {
const eventSource = new EventSource(event_url);
Expand Down
124 changes: 37 additions & 87 deletions client/src/pages/ProjectPage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ import { DropdownMenuText, DropdownOption } from "../components/DropDownMenus";
import { FileDropArea } from "../components/FileDropArea";
import { ExpandableToast } from "../components/ExpandableToast";
import { TruncatedFileNames } from "../components/TruncatedFileNames";
import { fetchJobTasksFromBackend } from "../services/jobTaskService";
import { createJob, fetchJobsForProject } from "../services/jobService";
import { createJob } from "../services/jobService";
import {
fileUploadToBackend,
fileFetchFromBackend,
Expand All @@ -17,9 +16,6 @@ import { ManualEvaluationModal } from "../components/ManualEvaluationModal";
import { Button } from "../components/Button";
import {
FetchedFile,
JobTask,
JobTaskStatus,
CreatedJob,
LlmConfig,
createZeroShotPromptingConfig,
JobPromptingType,
Expand Down Expand Up @@ -88,11 +84,10 @@ const ModelConfiguration: React.FC<ModelConfigurationProps> = ({
{Object.keys(modelParametersSchema.properties).map((key) => {
const property = modelParametersSchema.properties[key];
return (
<span key={`property_${property.title}`}>{`${property.title}: ${
modelFormValues[key] !== undefined &&
<span key={`property_${property.title}`}>{`${property.title}: ${modelFormValues[key] !== undefined &&
modelFormValues[key] !== "" &&
modelFormValues[key]
}`}</span>
}`}</span>
);
})}
</div>
Expand Down Expand Up @@ -124,7 +119,7 @@ const ModelConfiguration: React.FC<ModelConfigurationProps> = ({
</label>
<span className="text-sm font-medium text-slate-600">
{modelFormValues[key] !== undefined &&
modelFormValues[key] !== "" ? (
modelFormValues[key] !== "" ? (
<>{modelFormValues[key]}</>
) : (
""
Expand Down Expand Up @@ -236,8 +231,8 @@ const ProviderConfiguration: React.FC<ProviderConfigurationProps> = ({
</label>
<span className="text-sm font-medium text-slate-600">
{providerFormValues[key] !== undefined &&
property.type !== "string" &&
providerFormValues[key] !== "" ? (
property.type !== "string" &&
providerFormValues[key] !== "" ? (
<>{providerFormValues[key]}</>
) : (
""
Expand Down Expand Up @@ -413,7 +408,7 @@ export const ProjectPage = () => {
const [evaluateViewMatch] = useRoute("/project/:projectUuid/evaluate");
const [fewShotViewMatch] = useRoute("/project/:projectUuid/few_shot");
const search = useSearch();
const jobTaskRefetchIntervalMs = 5000;

const [isLlmProviderSelected, setIsLlmProviderSelected] = useState(false);
const [modelsLoaded, setModelsLoaded] = useState(false);
const [isLlmSelected, setIsLlmSelected] = useState(false);
Expand All @@ -422,12 +417,11 @@ export const ProjectPage = () => {
const getPapers = useTypedStoreState((state) => state.getPapersForProject);
const papers = getPapers(projectUuid);

const [createdJobs, setCreatedJobs] = useState<CreatedJob[]>([]);
const [fetchedFiles, setFetchedFiles] = useState<FetchedFile[]>([]);
const [jobTasks, setJobTasks] = useState<JobTask[]>([]);
const [availableModels, setAvailableModels] = useState<
Array<{ id: string; created: number; object: "model"; owned_by: string }>
>([]);

const loadingProjects = useTypedStoreState((state) => state.loading.projects);
const loadProjects = useTypedStoreActions((actions) => actions.fetchProjects);
const getProjectByUuid = useTypedStoreState(
Expand All @@ -436,6 +430,13 @@ export const ProjectPage = () => {
const providers = useTypedStoreState((state) => state.providers);
const fetchPapers = useTypedStoreActions((actions) => actions.fetchPapers);

const fetchJobsForProject = useTypedStoreActions(
(actions) => actions.fetchJobsForProject,
);
const jobs = useTypedStoreState(
(state) => state.jobsByProject[projectUuid] || [],
);

const project = getProjectByUuid(projectUuid);

useEffect(() => {
Expand All @@ -446,6 +447,12 @@ export const ProjectPage = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [project, projectUuid]);

useEffect(() => {
if (projectUuid) {
fetchJobsForProject(projectUuid);
}
}, [projectUuid, fetchJobsForProject]);

const paperUuid = useMemo(() => {
if (!search) return null;
return new URLSearchParams(search).get("paperUuid");
Expand Down Expand Up @@ -513,23 +520,7 @@ export const ProjectPage = () => {
[papers],
);

const evaluationFinished = jobTasks.length > 0 && pendingTasks.length === 0;

const fetchJobs = useCallback(() => {
async function doFetch() {
try {
const jobs = await fetchJobsForProject(projectUuid);
setCreatedJobs(jobs);
} catch (e) {
console.error("Failed to fetch jobs for project", e);
}
}
doFetch().catch(console.error);
}, [projectUuid]);

useEffect(() => {
fetchJobs();
}, [fetchJobs, projectUuid]);
const evaluationFinished = jobs.length === 0 && pendingTasks.length === 0;

const fetchModels = useCallback(() => {
async function fetch_models() {
Expand All @@ -545,7 +536,7 @@ export const ProjectPage = () => {
} catch (error) {
console.error(
"Failed to fetch available models for provider " +
selectedLlmProvider.value,
selectedLlmProvider.value,
error,
);
}
Expand All @@ -557,7 +548,7 @@ export const ProjectPage = () => {
const paperToTaskMap = useMemo(() => {
if (
papers.length === 0 ||
jobTasks.length === 0 ||
jobs.length === 0 ||
pendingTasks.length === 0
) {
return {};
Expand All @@ -579,7 +570,7 @@ export const ProjectPage = () => {
}
});
return map;
}, [papers, jobTasks, pendingTasks]);
}, [papers, jobs, pendingTasks]);

const currentTaskUuid = paperUuid ? paperToTaskMap[paperUuid] : undefined;

Expand All @@ -602,17 +593,8 @@ export const ProjectPage = () => {
const promptingConfig = createZeroShotPromptingConfig();

try {
const res = await createJob(projectUuid, llmConfig, promptingConfig);
const createdJob: CreatedJob = {
uuid: res.uuid,
project_uuid: res.project_uuid,
llm_config: res.llm_config,
prompting_config: res.prompting_config,
created_at: res.created_at,
updated_at: res.updated_at,
};
setCreatedJobs((prev) => [...prev, createdJob]);
// await loadPapers();
await createJob(projectUuid, llmConfig, promptingConfig);
fetchJobsForProject(projectUuid);
} catch (e) {
console.error("Error creating job:", e);
toast.error("Error creating job");
Expand All @@ -623,6 +605,7 @@ export const ProjectPage = () => {
modelFormValues,
providerFormValues,
projectUuid,
fetchJobsForProject,
]);

const uploadFilesToBackend = useCallback(
Expand Down Expand Up @@ -665,7 +648,6 @@ export const ProjectPage = () => {
try {
await uploadFilesToBackend(files);
await fetchFiles();
// await loadPapers();
} catch (error) {
console.error("Problem uploading the files", error);
}
Expand All @@ -687,31 +669,6 @@ export const ProjectPage = () => {
})();
}, [fetchFiles]);

useEffect(() => {
if (createdJobs.length === 0) return;

const fetchAll = () => {
Promise.all(
createdJobs.map((job) => {
// console.log("job.uuid", job.uuid);
// @ts-expect-error Expected
return fetchJobTasksFromBackend(job.uuid, job.id);
}),
)
.then((results) => {
setJobTasks(results.flat());
// console.log("results: ", results.flat());
})
.catch((error) => {
console.error("Error fetching job tasks:", error);
});
};

fetchAll();
const interval = setInterval(fetchAll, jobTaskRefetchIntervalMs);
return () => clearInterval(interval);
}, [createdJobs, jobTaskRefetchIntervalMs]);

const openManualEvaluation = useCallback(() => {
if (evaluationFinished) return;
if (papers.length === 0) {
Expand All @@ -730,25 +687,23 @@ export const ProjectPage = () => {
if (idx !== -1) {
for (let i = idx + 1; i < papers.length; i++) {
const candidate = papers[i];
if (jobTasks.length === 0 || paperToTaskMap[candidate.uuid]) {
if (jobs.length === 0 || paperToTaskMap[candidate.uuid]) {
navigate(
`/project/${projectUuid}/evaluate?paperUuid=${candidate.uuid}`,
);
return;
}
}
}
// await loadPapers();
navigate(`/project/${projectUuid}`);
toast.success("Manual evaluation finished.");
}, [
paperUuid,
papers,
jobTasks.length,
jobs.length,
paperToTaskMap,
navigate,
projectUuid,
// loadPapers,
]);

useEffect(() => {
Expand Down Expand Up @@ -835,19 +790,14 @@ export const ProjectPage = () => {
</div>
<div className="flex space-x-8 lg:flex-row flex-col items-start">
<div className="flex flex-col space-y-4 w-7xl">
{jobTasks.length === 0 && (
{jobs.length === 0 && (
<AlertMessage message="No screening tasks." />
)}
{createdJobs.map((job) => {
const tasks = jobTasks.filter((task) => task.job_uuid === job.uuid);
const doneCount = tasks.filter(
(task) => task.status === JobTaskStatus.DONE,
).length;
const errorCount = tasks.filter(
(task) => task.status === JobTaskStatus.ERROR,
).length;
const totalCount = tasks.length;
const completedCount = doneCount + errorCount;
{jobs.map((job) => {
const successCount = job.stats.success;
const errorCount = job.stats.failed;
const totalCount = job.stats.total;
const completedCount = successCount + errorCount;
const progress =
totalCount === 0
? 0
Expand Down Expand Up @@ -1140,7 +1090,7 @@ export const ProjectPage = () => {
}}
onClose={() => {
loadProjects();
fetchJobs();
fetchJobsForProject(projectUuid);
navigate(`/project/${projectUuid}`);
}}
/>
Expand Down
Loading