diff --git a/package-lock.json b/package-lock.json index e9fa1800e..276d73e42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,11 @@ "vscode-languageserver-protocol": "^3.16.0", "web-vitals": "^1.1.1", "xterm": "4.14.1", - "xterm-addon-fit": "^0.5.0" + "xterm-addon-fit": "^0.5.0", + "y-codemirror.next": "^0.3.5", + "y-indexeddb": "^9.0.12", + "y-protocols": "^1.0.6", + "yjs": "^13.6.27" }, "devDependencies": { "@babel/plugin-proposal-private-property-in-object": "^7.21.11", @@ -8633,6 +8637,16 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "license": "MIT", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/iterator.prototype": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", @@ -9394,6 +9408,27 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.114", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.114.tgz", + "integrity": "sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ==", + "license": "MIT", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -12895,6 +12930,64 @@ "xterm": "^4.0.0" } }, + "node_modules/y-codemirror.next": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/y-codemirror.next/-/y-codemirror.next-0.3.5.tgz", + "integrity": "sha512-VluNu3e5HfEXybnypnsGwKAj+fKLd4iAnR7JuX1Sfyydmn1jCBS5wwEL/uS04Ch2ib0DnMAOF6ZRR/8kK3wyGw==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.42" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "yjs": "^13.5.6" + } + }, + "node_modules/y-indexeddb": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.12.tgz", + "integrity": "sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.74" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "node_modules/y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", @@ -12914,6 +13007,23 @@ "node": ">= 14.6" } }, + "node_modules/yjs": { + "version": "13.6.27", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", + "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/zod": { "version": "3.24.2", "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz", diff --git a/package.json b/package.json index 3957f72b2..aac511fc0 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,11 @@ "vscode-languageserver-protocol": "^3.16.0", "web-vitals": "^1.1.1", "xterm": "4.14.1", - "xterm-addon-fit": "^0.5.0" + "xterm-addon-fit": "^0.5.0", + "y-codemirror.next": "^0.3.5", + "y-indexeddb": "^9.0.12", + "y-protocols": "^1.0.6", + "yjs": "^13.6.27" }, "devDependencies": { "@babel/plugin-proposal-private-property-in-object": "^7.21.11", diff --git a/src/App.tsx b/src/App.tsx index 813fea79d..8a7788e20 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -30,6 +30,8 @@ import SettingsProvider from "./settings/settings"; import BeforeUnloadDirtyCheck from "./workbench/BeforeUnloadDirtyCheck"; import { SelectionProvider } from "./workbench/use-selection"; import Workbench from "./workbench/Workbench"; +import { ProjectStorageProvider } from "./project-persistence/ProjectStorageProvider"; +import ProjectPageRouting from "./ProjectPageRouting"; const isMockDeviceMode = () => // We use a cookie set from the e2e tests. Avoids having separate test and live builds. @@ -79,15 +81,19 @@ const App = () => { - - - - - - - - - + + + + + + + + + + + + + diff --git a/src/ProjectPageRouting.tsx b/src/ProjectPageRouting.tsx new file mode 100644 index 000000000..5be3f0ba3 --- /dev/null +++ b/src/ProjectPageRouting.tsx @@ -0,0 +1,34 @@ +import { ReactNode, useEffect } from "react"; +import { useRouterState } from "./router-hooks"; +import ProjectBrowser from "./project/ProjectBrowser"; +import { useProjectList } from "./project-persistence/project-list-hooks"; +import { usePersistentProject } from "./project-persistence/persistent-project-hooks"; + +interface ProjectPageRoutingProps { + children: ReactNode; +} +const ProjectPageRouting = ({ children }: ProjectPageRoutingProps) => { + const [{ tab }] = useRouterState(); + const { projectList, restoreStoredProject } = useProjectList(); + const { projectId } = usePersistentProject(); + + useEffect(() => { + if (!projectId && projectList) { + const restoreState = async () => { + const restoredProject = await restoreStoredProject(projectList[0].id); + if (!restoredProject && typeof tab !== "undefined") { + history.replaceState(null, "", "/"); + window.dispatchEvent(new PopStateEvent("popstate")); + } + }; + void restoreState(); + } + }, [projectId, projectList, restoreStoredProject, tab]); + + if (typeof tab === "undefined") { + return ; + } + return children; +}; + +export default ProjectPageRouting; diff --git a/src/editor/EditorContainer.tsx b/src/editor/EditorContainer.tsx index bb21a25ac..1fc8e9f85 100644 --- a/src/editor/EditorContainer.tsx +++ b/src/editor/EditorContainer.tsx @@ -9,6 +9,8 @@ import { useSettings } from "../settings/settings"; import { WorkbenchSelection } from "../workbench/use-selection"; import Editor from "./codemirror/CodeMirror"; import ModuleOverlay from "./ModuleOverlay"; +import { usePersistentProject } from "../project-persistence/persistent-project-hooks"; +import * as Y from "yjs"; interface EditorContainerProps { selection: WorkbenchSelection; @@ -25,16 +27,27 @@ const EditorContainer = ({ selection }: EditorContainerProps) => { }, [setSettings, settings]); // Note fileInfo is not updated for ordinary text edits. const [fileInfo, onFileChange] = useProjectFileText(selection.file); + const { ydoc, awareness } = usePersistentProject(); + + const ytext = ydoc?.getMap("files").get(selection.file) as Y.Text; + + if (ytext === null) return null; if (fileInfo === undefined) { return null; } + awareness!.setLocalStateField("user", { + name: "micro:bit tester", + color: "yellow", + }); + + // TODO: represent fileInfo in project? + return fileInfo.isThirdPartyModule && !settings.allowEditingThirdPartyModules ? ( ) : ( { parameterHelpOption={settings.parameterHelp} warnOnV2OnlyFeatures={settings.warnForApiUnsupportedByDevice} disableV2OnlyFeaturesWarning={disableV2OnlyFeaturesWarning} + awareness={awareness!} + text={ytext} /> ); }; diff --git a/src/editor/codemirror/CodeMirror.tsx b/src/editor/codemirror/CodeMirror.tsx index e2aa339a6..9ac4a9a67 100644 --- a/src/editor/codemirror/CodeMirror.tsx +++ b/src/editor/codemirror/CodeMirror.tsx @@ -41,12 +41,15 @@ import { lintGutter } from "./lint/lint"; import { codeStructure } from "./structure-highlighting"; import themeExtensions from "./themeExtensions"; import { useDevice } from "../../device/device-hooks"; +import * as Y from "yjs"; +import { Awareness } from 'y-protocols/awareness.js' +import { yCollab } from "y-codemirror.next"; interface CodeMirrorProps { className?: string; - defaultValue: string; onChange: (doc: string) => void; - + text: Y.Text; + awareness: Awareness; selection: WorkbenchSelection; fontSize: number; codeStructureOption: CodeStructureOption; @@ -64,9 +67,10 @@ interface CodeMirrorProps { * (e.g. based on the file being edited). */ const CodeMirror = ({ - defaultValue, className, onChange, + text, + awareness, selection, fontSize, codeStructureOption, @@ -87,6 +91,7 @@ const CodeMirror = ({ const [sessionSettings, setSessionSettings] = useSessionSettings(); const { apiReferenceMap } = useDocumentation(); const device = useDevice(); + const textRef = useRef(); // Reset undo/redo events on file change. useEffect(() => { @@ -108,8 +113,14 @@ const CodeMirror = ({ ); useEffect(() => { - const initializing = !viewRef.current; + let initializing = !viewRef.current; + // Recreate if the text doc changes + if (!initializing && textRef.current !== text) { + elementRef.current?.replaceChildren(); + initializing = true; + } if (initializing) { + textRef.current = text; const notify = EditorView.updateListener.of((update) => { if (update.docChanged) { onChange(update.state.sliceDoc(0)); @@ -121,8 +132,9 @@ const CodeMirror = ({ } }); const state = EditorState.create({ - doc: defaultValue, + doc: text.toString(), extensions: [ + yCollab(text, awareness), notify, editorConfig, // Extension requires external state. @@ -165,14 +177,13 @@ const CodeMirror = ({ state, parent: elementRef.current!, }); - viewRef.current = view; setActiveEditor(new EditorActions(view, logging, actionFeedback, intl)); } }, [ + awareness, actionFeedback, client, - defaultValue, intl, logging, onChange, @@ -186,6 +197,7 @@ const CodeMirror = ({ apiReferenceMap, device, disableV2OnlyFeaturesWarning, + text, ]); useEffect(() => { // Do this separately as we don't want to destroy the view whenever options needed for initialization change. diff --git a/src/fs/fs.ts b/src/fs/fs.ts index ba93f24e5..420dd518b 100644 --- a/src/fs/fs.ts +++ b/src/fs/fs.ts @@ -364,6 +364,18 @@ export class FileSystem extends TypedEventTarget { await this.replaceCommon(project.projectName); } + async replaceWithMultipleStrings(project: { + files: Record, + projectName: string + }): Promise { + const fs = await this.initialize(); + fs.ls().forEach((f) => fs.remove(f)); + for (const key in project.files) { + fs.write(key, new TextEncoder().encode(project.files[key])); + } + await this.replaceCommon(project.projectName); + } + async replaceWithHexContents( projectName: string, hex: string diff --git a/src/project-persistence/ProjectHistoryModal.tsx b/src/project-persistence/ProjectHistoryModal.tsx new file mode 100644 index 000000000..be6a397ee --- /dev/null +++ b/src/project-persistence/ProjectHistoryModal.tsx @@ -0,0 +1,87 @@ +import { Modal, ModalOverlay, ModalContent, ModalHeader, ModalCloseButton, ModalBody, VStack, List, ListItem, Heading, Button, ModalFooter } from "@chakra-ui/react"; +import { HistoryList } from "./project-history-db"; +import { ProjectEntry } from "./project-list-db"; +import { useCallback, useEffect, useState } from "react"; +import { significantDateUnits } from "./utils"; +import { useProjectHistory } from "./project-history-hooks"; + +interface ProjectHistoryModalProps { + onLoadRequest: (projectId: string, revisionId: string) => void; + isOpen: boolean; + onDismiss: () => void; + projectInfo: ProjectEntry | null; +} + +const ProjectHistoryModal = ({ + onLoadRequest, + isOpen, + onDismiss, + projectInfo, +}: ProjectHistoryModalProps) => { + const [projectHistoryList, setProjectHistoryList] = + useState(null); + const { getHistory, saveRevision } = useProjectHistory(); + + const getProjectHistory = useCallback(async () => { + if (projectInfo === null) { + setProjectHistoryList(null); + return; + } + const historyList = await getHistory(projectInfo.id); + setProjectHistoryList(historyList.sort((h) => -h.timestamp)); + }, [getHistory, projectInfo]); + + useEffect(() => { + void getProjectHistory(); + }, [projectInfo, getProjectHistory]); + + return ( + + + + Project history + + + {projectInfo && ( + + {projectInfo.projectName} + + + Latest + + + {projectHistoryList?.map((ph) => ( + + + Saved on {significantDateUnits(new Date(ph.timestamp))} + + + + ))} + + + )} + + + + + + + + ); +}; + +export default ProjectHistoryModal; \ No newline at end of file diff --git a/src/project-persistence/ProjectItem.tsx b/src/project-persistence/ProjectItem.tsx new file mode 100644 index 000000000..c1a4be901 --- /dev/null +++ b/src/project-persistence/ProjectItem.tsx @@ -0,0 +1,114 @@ +import { + CloseButton, + GridItem, + Heading, + HStack, + IconButton, + Text, +} from "@chakra-ui/react"; +import { ReactNode } from "react"; +import { ProjectEntry } from "./project-list-db"; +import { timeAgo } from "./utils"; +import { RiEditFill, RiHistoryFill } from "react-icons/ri"; + +interface ProjectItemProps { + project: ProjectEntry; + showHistory: (projectId: string) => void; + loadProject: (projectId: string) => void; + deleteProject: (projectId: string) => void; + renameProject: (projectId: string) => void; +} + +interface ProjectItemBaseProps { + children: ReactNode; + onClick: () => void; +} + +const ProjectItemBase = ({ onClick, children }: ProjectItemBaseProps) => ( + + {children} + +); + +export const ProjectItem = ({ + project, + loadProject, + deleteProject, + renameProject, + showHistory, +}: ProjectItemProps) => ( + loadProject(project.id)}> + + + {project.projectName} + + + {timeAgo(new Date(project.modifiedDate))} + + } + mr="2" + onClick={(e) => { + showHistory(project.id); + e.stopPropagation(); + e.preventDefault(); + }} + size="lg" + title="Project history" + variant="outline" + /> + } + onClick={(e) => { + renameProject(project.id); + e.stopPropagation(); + e.preventDefault(); + }} + size="lg" + title="Rename" + variant="outline" + /> + { + deleteProject(project.id); + e.stopPropagation(); + e.preventDefault(); + }} + /> + +); + +interface AddProjectItemProps { + newProject: () => void; +} + +export const AddProjectItem = ({ newProject }: AddProjectItemProps) => ( + + + New project + + Click to create + +); diff --git a/src/project-persistence/ProjectStorageProvider.tsx b/src/project-persistence/ProjectStorageProvider.tsx new file mode 100644 index 000000000..5a2e3b8da --- /dev/null +++ b/src/project-persistence/ProjectStorageProvider.tsx @@ -0,0 +1,69 @@ +// ProjectContext.tsx +import React, { createContext, useCallback, useContext, useState } from "react"; +import { ProjectList } from "./project-list-db"; +import { ProjectStore } from "./project-store"; + +interface ProjectContextValue { + projectId: string | null; + projectList: ProjectList | null; + setProjectList: (projectList: ProjectList) => void; + projectStore: ProjectStore | null; + setProjectStore: (projectStore: ProjectStore) => void; +} + +export const ProjectStorageContext = createContext( + null +); + +/** + * The ProjectStorageProvider is intended to be used only through the hooks in + * + * - project-list-hooks.ts: information about hooks that does not require an open project + * - persistent-project-hooks.ts: manages a currently open project + * - project-history-hooks.ts: manages project history and revisions + * + * This structure is helpful for working out what parts of project persistence are used + * where. + */ +export function ProjectStorageProvider({ + children, +}: { + children: React.ReactNode; +}) { + const [projectList, setProjectList] = useState(null); + const [projectStore, setProjectStoreImpl] = useState( + null + ); + const setProjectStore = useCallback( + (newProjectStore: ProjectStore) => { + if (projectStore) { + projectStore.destroy(); + } + setProjectStoreImpl(newProjectStore); + }, + [projectStore] + ); + + return ( + + {children} + + ); +} + +export function useProjectStorage() { + const ctx = useContext(ProjectStorageContext); + if (!ctx) + throw new Error( + "useProjectStorage must be used within a ProjectStorageProvider" + ); + return ctx; +} diff --git a/src/project-persistence/RenameProjectModal.tsx b/src/project-persistence/RenameProjectModal.tsx new file mode 100644 index 000000000..003752329 --- /dev/null +++ b/src/project-persistence/RenameProjectModal.tsx @@ -0,0 +1,61 @@ +import { Modal, ModalOverlay, ModalContent, ModalHeader, ModalCloseButton, ModalBody, VStack, Button, ModalFooter, Input } from "@chakra-ui/react"; +import { ProjectEntry } from "./project-list-db"; +import { useEffect, useState } from "react"; + +interface ProjectHistoryModalProps { + handleRename: (projectId: string, projectName: string) => void; + isOpen: boolean; + onDismiss: () => void; + projectInfo: ProjectEntry | null; +} + +const RenameProjectModal = ({ + handleRename, + isOpen, + onDismiss, + projectInfo +}: ProjectHistoryModalProps) => { + const [projectName, setProjectName] = useState(projectInfo?.projectName || ""); + + useEffect(() => { + if (!projectInfo) { + return; + } + setProjectName(projectInfo.projectName); + }, [projectInfo]); + + return ( + + + Project history + + + {projectInfo && ( + + setProjectName(e.target.value)} /> + )} + + + + + + + + ) + } + +export default RenameProjectModal; \ No newline at end of file diff --git a/src/project-persistence/persistent-project-hooks.ts b/src/project-persistence/persistent-project-hooks.ts new file mode 100644 index 000000000..199a13ae6 --- /dev/null +++ b/src/project-persistence/persistent-project-hooks.ts @@ -0,0 +1,24 @@ +import { useContext } from "react"; +import { ProjectStorageContext } from "./ProjectStorageProvider"; +import * as Y from "yjs"; +import { Awareness } from "y-protocols/awareness.js"; + +interface PersistentProjectActions { + ydoc?: Y.Doc; + awareness?: Awareness; + projectId?: string; +} + +export const usePersistentProject = (): PersistentProjectActions => { + + const ctx = useContext(ProjectStorageContext); + if (!ctx) + throw new Error( + "usePersistentProject must be used within a ProjectStorageProvider" + ); + return { + ydoc: ctx.projectStore?.ydoc, + awareness: ctx.projectStore?.awareness, + projectId: ctx.projectStore?.projectId + }; +} diff --git a/src/project-persistence/project-history-db.ts b/src/project-persistence/project-history-db.ts new file mode 100644 index 000000000..10764cf7b --- /dev/null +++ b/src/project-persistence/project-history-db.ts @@ -0,0 +1,57 @@ + +export interface HistoryEntry { + projectId: string; + revisionId: string; + parentId: string; + data: Uint8Array; + timestamp: number; +} + +export type HistoryList = HistoryEntry[]; + +type HistoryDbWrapper = ( + accessMode: "readonly" | "readwrite", + callback: (revisions: IDBObjectStore) => Promise +) => Promise; + +export const withHistoryDb: HistoryDbWrapper = async (accessMode, callback) => { + return new Promise((res, rej) => { + const openRequest = indexedDB.open("UserProjectHistory", 1); + openRequest.onupgradeneeded = (evt: IDBVersionChangeEvent) => { + const db = openRequest.result; + const tx = (evt.target as IDBOpenDBRequest).transaction; + + let revisions: IDBObjectStore; + if (!db.objectStoreNames.contains("revisions")) { + revisions = db.createObjectStore("revisions", { autoIncrement:true }); + } else { + revisions = tx!.objectStore("revisions"); + } + if (!revisions.indexNames.contains("projectRevision")) { + revisions.createIndex("projectRevision", ["projectId", "revisionId"]); + } + if (!revisions.indexNames.contains("projectParent")) { + revisions.createIndex("projectParent", ["projectId", "parentId"]); + } + if (!revisions.indexNames.contains("projectId")) { + revisions.createIndex("projectId", "projectId"); + } + }; + + openRequest.onsuccess = async () => { + const db = openRequest.result; + + const tx = db.transaction("revisions", accessMode); + const store = tx.objectStore("revisions"); + tx.onabort = rej; + tx.onerror = rej; + + const result = await callback(store); + + // got the result, but don't return until the transaction is complete + tx.oncomplete = () => res(result); + }; + + openRequest.onerror = rej; + }); +}; diff --git a/src/project-persistence/project-history-hooks.ts b/src/project-persistence/project-history-hooks.ts new file mode 100644 index 000000000..8e416f5fb --- /dev/null +++ b/src/project-persistence/project-history-hooks.ts @@ -0,0 +1,115 @@ +import { useCallback, useContext } from "react"; +import { ProjectStorageContext } from "./ProjectStorageProvider"; +import { HistoryEntry, HistoryList, withHistoryDb } from "./project-history-db"; +import { modifyProject, ProjectEntry, withProjectDb } from "./project-list-db"; +import { makeUID } from "./utils"; +import * as Y from "yjs"; +import { ProjectStore } from "./project-store"; +import { useProjectList } from "./project-list-hooks"; + +/** + * Each project has a "head" which is a Y.Doc, and a series of revisions which are Y.js Update deltas. + */ +interface ProjectHistoryActions { + getHistory: (projectId: string) => Promise; + /** + * Note that loading a revision creates a new instance of the project at that revision. + * + * TODO: if a user loads a revision and doesn't modify it, should we even keep it around? + */ + loadRevision: (projectId: string, projectRevision: string) => Promise; + /** + * Converts the head of the given project into a revision. + * + * TODO: prevent creating empty revisions if nothing changes. + */ + saveRevision: (projectInfo: ProjectEntry) => Promise; +} + +export const useProjectHistory = (): ProjectHistoryActions => { + const ctx = useContext(ProjectStorageContext); + if (!ctx) { + throw new Error( + "useProjectHistory must be used within a ProjectStorageProvider" + ); + } + const { newStoredProject } = useProjectList(); + + const getUpdateAtRevision = useCallback(async (projectId: string, revision: string) => { + const deltas: HistoryEntry[] = []; + let parentRevision = revision; + do { + const delta = await withHistoryDb("readonly", async (revisions) => { + return new Promise((res, _rej) => { + const query = revisions + .index("projectRevision") + .get([projectId, parentRevision]); + query.onsuccess = () => res(query.result as HistoryEntry); + }); + }); + parentRevision = delta.parentId; + deltas.unshift(delta); + } while (parentRevision); + return Y.mergeUpdatesV2(deltas.map((d) => d.data)); + }, []); + + const getProjectInfo = (projectId: string) => + withProjectDb("readwrite", async (store) => { + return new Promise((res, _rej) => { + const query = store.get(projectId); + query.onsuccess = () => res(query.result as ProjectEntry); + }); + }); + + const loadRevision = useCallback(async (projectId: string, projectRevision: string) => { + const projectInfo = await getProjectInfo(projectId); + const { ydoc, id: forkId } = await newStoredProject(); + await modifyProject(forkId, { + projectName: `${projectInfo.projectName} revision`, + parentRevision: forkId, + }); + const updates = await getUpdateAtRevision(projectId, projectRevision); + Y.applyUpdateV2(ydoc, updates); + }, [getUpdateAtRevision, newStoredProject]); + + const saveRevision = useCallback(async (projectInfo: ProjectEntry) => { + const projectStore = new ProjectStore(projectInfo.id, () => { }); + await projectStore.persist(); + let newUpdate: Uint8Array; + if (projectInfo.parentRevision) { + const previousUpdate = await getUpdateAtRevision( + projectInfo.id, + projectInfo.parentRevision + ); + newUpdate = Y.encodeStateAsUpdateV2(projectStore.ydoc, previousUpdate); + } else { + newUpdate = Y.encodeStateAsUpdateV2(projectStore.ydoc); + } + const newRevision = makeUID(); + await withHistoryDb("readwrite", async (revisions) => { + return new Promise((res, _rej) => { + const query = revisions.put({ + projectId: projectInfo.id, + revisionId: newRevision, + parentId: projectInfo.parentRevision, + data: newUpdate, + timestamp: new Date(), + }); + query.onsuccess = () => res(); + }); + }); + await modifyProject(projectInfo.id, { parentRevision: newRevision }); + }, [getUpdateAtRevision]); + + const getHistory = useCallback(async (projectId: string) => + withHistoryDb("readonly", async (store) => { + const revisionList = await new Promise((res, _rej) => { + const query = store.index("projectId").getAll(projectId); + query.onsuccess = () => res(query.result); + }); + return revisionList; + }), []); + + + return { getHistory, loadRevision, saveRevision }; +} diff --git a/src/project-persistence/project-list-db.ts b/src/project-persistence/project-list-db.ts new file mode 100644 index 000000000..3a937e70e --- /dev/null +++ b/src/project-persistence/project-list-db.ts @@ -0,0 +1,82 @@ + +export interface ProjectEntry { + projectName: string; + id: string; + modifiedDate: number; + parentRevision?: string; +} + +export type ProjectList = ProjectEntry[]; + +type ProjectDbWrapper = ( + accessMode: "readonly" | "readwrite", + callback: (projects: IDBObjectStore) => Promise +) => Promise; + +export const withProjectDb: ProjectDbWrapper = async (accessMode, callback) => { + return new Promise((res, rej) => { + // TODO: what if multiple users? I think MakeCode just keeps everything... + const openRequest = indexedDB.open("UserProjects", 2); + openRequest.onupgradeneeded = (evt: IDBVersionChangeEvent) => { + const db = openRequest.result; + // NB: a more robust way to write migrations would be to get the current stored + // db.version and open it repeatedly with an ascending version number until the + // db is up to date. That would be more boilerplate though. + const tx = (evt.target as IDBOpenDBRequest).transaction; + // if the data object store doesn't exist, create it + + let projects: IDBObjectStore; + if (!db.objectStoreNames.contains("projects")) { + projects = db.createObjectStore("projects", { keyPath: "id" }); + // no indexes at present, get the whole db each time + } else { + projects = tx!.objectStore("projects"); + } + if (!projects.indexNames.contains("modifiedDate")) { + projects.createIndex("modifiedDate", "modifiedDate"); + const now = new Date().valueOf(); + const updateProjectData = projects.getAll(); + updateProjectData.onsuccess = () => { + updateProjectData.result.forEach((project) => { + if (!('modifiedDate' in project)) { + projects.put({ ...project, modifiedDate: now }); + } + }); + }; + } + }; + + openRequest.onsuccess = async () => { + const db = openRequest.result; + + const tx = db.transaction("projects", accessMode); + const store = tx.objectStore("projects"); + tx.onabort = rej; + tx.onerror = rej; + + const result = await callback(store); + + // got the result, but don't return until the transaction is complete + tx.oncomplete = () => res(result); + }; + + openRequest.onerror = rej; + }); +}; + + +export const modifyProject = async (id: string, extras?: Partial) => { + await withProjectDb("readwrite", async (store) => { + await new Promise((res, _rej) => { + const getQuery = store.get(id); + getQuery.onsuccess = () => { + const putQuery = store.put({ + ...getQuery.result, + ...extras, + modifiedDate: new Date().valueOf(), + }); + putQuery.onsuccess = () => res(getQuery.result); + }; + }); + }); +} diff --git a/src/project-persistence/project-list-hooks.ts b/src/project-persistence/project-list-hooks.ts new file mode 100644 index 000000000..bd8cb0681 --- /dev/null +++ b/src/project-persistence/project-list-hooks.ts @@ -0,0 +1,120 @@ +import { useCallback, useContext, useEffect } from "react"; +import { ProjectStorageContext } from "./ProjectStorageProvider"; +import * as Y from "yjs"; +import { ProjectStore } from "./project-store"; +import { modifyProject, ProjectList, withProjectDb } from "./project-list-db"; +import { makeUID } from "./utils"; + +export interface NewStoredDoc { + id: string; + ydoc: Y.Doc; +} + +export interface RestoredStoredDoc { + projectName: string; + ydoc: Y.Doc; +} + +interface ProjectListActions { + newStoredProject: () => Promise; + restoreStoredProject: (id: string) => Promise; + deleteProject: (id: string) => Promise; + setProjectName: (id: string, name: string) => Promise; + projectList: ProjectList | null; +} + +export const useProjectList = (): ProjectListActions => { + + const ctx = useContext(ProjectStorageContext); + + if (!ctx) { + throw new Error( + "useProjectList must be used within a ProjectStorageProvider" + ); + } + + const { setProjectList, projectList, setProjectStore } = ctx; + + const refreshProjects = useCallback(async () => { + const projectList = await withProjectDb("readonly", async (store) => { + const projectList = await new Promise((res, _rej) => { + const query = store.index("modifiedDate").getAll(); + query.onsuccess = () => res(query.result); + }); + return projectList; + }); + setProjectList((projectList as ProjectList).reverse()); + }, [setProjectList]); + + useEffect(() => { + if (window.navigator.storage?.persist) { + void window.navigator.storage.persist(); + } + void refreshProjects(); + }, [refreshProjects]); + + const setProjectName = useCallback( + async (id: string, projectName: string) => { + await modifyProject(id, { projectName }); + await refreshProjects(); + }, + [refreshProjects] + ); + + const restoreStoredProject: ( + projectId: string + ) => Promise = useCallback( + async (projectId: string) => { + const newProjectStore = new ProjectStore(projectId, () => + modifyProject(projectId) + ); + await newProjectStore.persist(); + newProjectStore.startSyncing(); + setProjectStore(newProjectStore); + return { + ydoc: newProjectStore.ydoc, + projectName: projectList!.find((prj) => prj.id === projectId)! + .projectName, + }; + }, + [projectList, setProjectStore] + ); + + const newStoredProject: () => Promise = + useCallback(async () => { + const newProjectId = makeUID(); + await withProjectDb("readwrite", async (store) => { + store.add({ + id: newProjectId, + projectName: "Untitled project", + modifiedDate: new Date().valueOf(), + }); + return Promise.resolve(); + }); + const newProjectStore = new ProjectStore(newProjectId, () => + modifyProject(newProjectId) + ); + await newProjectStore.persist(); + newProjectStore.startSyncing(); + setProjectStore(newProjectStore); + return { ydoc: newProjectStore.ydoc, id: newProjectId }; + }, [ setProjectStore]); + + const deleteProject: (id: string) => Promise = useCallback( + async (id) => { + await withProjectDb("readwrite", async (store) => { + store.delete(id); + return refreshProjects(); + }); + }, + [refreshProjects] + ); + + return { + restoreStoredProject, + newStoredProject, + deleteProject, + setProjectName, + projectList + }; +} diff --git a/src/project-persistence/project-store.ts b/src/project-persistence/project-store.ts new file mode 100644 index 000000000..813b45172 --- /dev/null +++ b/src/project-persistence/project-store.ts @@ -0,0 +1,78 @@ +import { IndexeddbPersistence } from "y-indexeddb"; +import { Awareness } from "y-protocols/awareness.js"; +import * as Y from "yjs"; + +/** + * Because the ydoc persistence/sync needs to clean itself up from time to time + * it is in a class with the following state. It is agnostic in itself whether the project with the + * specified UID exists. + * + * constructor - sets up the state + * init - connects the persistence store, and local sync broadcast. Asynchronous, so you can await it + * destroy - disconnects everything that was connected in init, cleans up the persistence store + */ +export class ProjectStore { + public ydoc: Y.Doc; + public awareness: Awareness; + private broadcastHandler: (e: MessageEvent) => void; + private persistence: IndexeddbPersistence; + private updates: BroadcastChannel; + private updatePoster: (update: Uint8Array) => void; + + constructor(public projectId: string, projectChangedListener: () => void) { + const ydoc = new Y.Doc(); + this.ydoc = ydoc; + this.awareness = new Awareness(this.ydoc); + + this.persistence = new IndexeddbPersistence(this.projectId, this.ydoc); + + const clientId = `${Math.random()}`; // Used by the broadcasthandler to know whether we sent a data update + this.broadcastHandler = ({ data }: MessageEvent) => { + if (data.clientId !== clientId && data.projectId === projectId) { + Y.applyUpdate(ydoc, data.update); + } + }; + + this.updates = new BroadcastChannel("yjs"); + this.updatePoster = ((update: Uint8Array) => { + this.updates.postMessage({ clientId, update, projectId }); + projectChangedListener(); + }).bind(this); + } + + public async persist() { + await new Promise((res) => this.persistence.once("synced", res)); + migrate(this.ydoc); + } + + public startSyncing() { + this.ydoc.on("update", this.updatePoster); + this.updates.addEventListener("message", this.broadcastHandler); + } + + public destroy() { + this.ydoc.off("update", this.updatePoster); + this.updates.removeEventListener("message", this.broadcastHandler); + this.updates.close(); + void this.persistence.destroy(); + } +} + +/** + * This is a kind of example of what migration could look like. It's not a designed approach at this point. + */ +const migrate = (doc: Y.Doc) => { + const meta = doc.getMap("meta"); + if (!meta.has("version")) { + // If the project has no version, assume it's from whatever this app did before ProjectStorageProvider + // This could be a per-app handler + meta.set("version", 1); + meta.set("projectName", "default"); // TODO: get this from the last loaded project name + } +}; + +interface SyncMessage { + clientId: string; + projectId: string; + update: Uint8Array; +} \ No newline at end of file diff --git a/src/project-persistence/utils.ts b/src/project-persistence/utils.ts new file mode 100644 index 000000000..bc3a774cf --- /dev/null +++ b/src/project-persistence/utils.ts @@ -0,0 +1,51 @@ +export function timeAgo(date: Date): string { + const now = new Date(); + const seconds = Math.round((+now - +date) / 1000); + const rtf = new Intl.RelativeTimeFormat('en', { numeric: 'auto' }); + + const divisions: { amount: number; unit: Intl.RelativeTimeFormatUnit }[] = [ + { amount: 60, unit: 'second' }, + { amount: 60, unit: 'minute' }, + { amount: 24, unit: 'hour' }, + { amount: 7, unit: 'day' }, + { amount: 4.34524, unit: 'week' }, // approx + { amount: 12, unit: 'month' } + ]; + + let duration = seconds; + for (const division of divisions) { + if (Math.abs(duration) < division.amount) { + return rtf.format(-Math.round(duration), division.unit); + } + duration /= division.amount; + } + + return rtf.format(-Math.round(duration), "year"); +} + +export function significantDateUnits(date: Date): string { + const now = new Date(); + + let dateTimeOptions: Intl.DateTimeFormatOptions = { month: "short", year: "2-digit" }; + + const daysDifferent = Math.round((+now - +date) / (1000 * 60 * 60 * 24)); + if (daysDifferent < 1 && date.getDay() === now.getDay()) { + dateTimeOptions = { + hour: 'numeric', + minute: 'numeric', + } + } else if (now.getFullYear() === date.getFullYear()) { + dateTimeOptions = { + day: 'numeric', + month: 'short' + } + } + + return Intl.DateTimeFormat(undefined, dateTimeOptions).format(date); +} + +// TODO: WORLDS UGLIEST UIDS +export const makeUID = () => { + return `${Math.random()}`; +}; + diff --git a/src/project/ProjectBrowser.tsx b/src/project/ProjectBrowser.tsx new file mode 100644 index 000000000..ad49bd6e3 --- /dev/null +++ b/src/project/ProjectBrowser.tsx @@ -0,0 +1,78 @@ +import { Grid } from "@chakra-ui/react"; +import { useProjectStorage } from "../project-persistence/ProjectStorageProvider"; +import { useRouterState } from "../router-hooks"; +import { useProjectActions } from "./project-hooks"; +import { + ProjectItem, + AddProjectItem, +} from "../project-persistence/ProjectItem"; +import RenameProjectModal from "../project-persistence/RenameProjectModal"; +import { useState } from "react"; +import { ProjectEntry } from "../project-persistence/project-list-db"; +import ProjectHistoryModal from "../project-persistence/ProjectHistoryModal"; +import { useProjectList } from "../project-persistence/project-list-hooks"; +import { useProjectHistory } from "../project-persistence/project-history-hooks"; + +const ProjectBrowser = () => { + const { projectList } = useProjectStorage(); + const { deleteProject, setProjectName } = useProjectList(); + const { loadRevision } = useProjectHistory(); + + const { newProject, loadProject } = useProjectActions(); + const [_, setParams] = useRouterState(); + const [showProjectHistory, setShowProjectHistory] = + useState(null); + const [showProjectRename, setShowProjectRename] = + useState(null); + + return ( + <> + + { + await newProject(); + setParams({ tab: "project" }); + }} + /> + {projectList?.map((proj) => ( + { + await loadProject(proj.id); + setParams({ tab: "project" }); + }} + deleteProject={deleteProject} + renameProject={() => setShowProjectRename(proj)} + showHistory={() => setShowProjectHistory(proj)} + /> + ))} + + { + await loadRevision(projectId, revisionId); + setParams({ tab: "project" }); + }} + onDismiss={() => setShowProjectHistory(null)} + projectInfo={showProjectHistory} + /> + setShowProjectRename(null)} + projectInfo={showProjectRename} + handleRename={(projectId, projectName) => { + setProjectName(projectId, projectName); + setShowProjectRename(null); + }} + /> + + ); +}; + +export default ProjectBrowser; diff --git a/src/project/project-actions.tsx b/src/project/project-actions.tsx index 6ac24c17e..525ec2a8d 100644 --- a/src/project/project-actions.tsx +++ b/src/project/project-actions.tsx @@ -71,6 +71,12 @@ import ProjectNameQuestion from "./ProjectNameQuestion"; import WebUSBErrorDialog from "../workbench/connect-dialogs/WebUSBErrorDialog"; import reconnectWebm from "../workbench/connect-dialogs/reconnect.webm"; import reconnectMp4 from "../workbench/connect-dialogs/reconnect.mp4"; +import * as Y from "yjs"; +import { toByteArray } from "base64-js"; +import { + NewStoredDoc, + RestoredStoredDoc, +} from "../project-persistence/project-list-hooks"; /** * Distinguishes the different ways to trigger the load action. @@ -119,7 +125,14 @@ export class ProjectActions { }, private intl: IntlShape, private logging: Logging, - private client: LanguageServerClient | undefined + private client: LanguageServerClient | undefined, + private ydoc: Y.Doc | null, + private projectId: string | null, + private newStoredProject: () => Promise, + private restoreStoredProject: ( + projectId: string + ) => Promise, + private setStoredProjectName: (id: string, name: string) => void ) {} private get project(): DefaultedProject { @@ -369,6 +382,10 @@ export class ProjectActions { /** * Open a project, asking for confirmation if required. + * Used for openIdea, open default project on reset, not used for localstorage. + * + * Just pass in a ydoc, because there are times when a user might + * reset their project without creating a new project * * @param project The project. * @param confirmPrompt Optional custom confirmation prompt. @@ -376,15 +393,33 @@ export class ProjectActions { */ private openProject = async ( project: PythonProject, + ydoc: Y.Doc, confirmPrompt?: string ): Promise => { const confirmed = await this.confirmReplace(confirmPrompt); if (confirmed) { await this.fs.replaceWithMultipleFiles(project); + const files = ydoc.getMap("files"); + files.clear(); + + const decoder = new TextDecoder(); + for (const fileName of Object.keys(project.files)) { + let text = project.files[fileName]; + text = decoder.decode(toByteArray(text)); + const ytext = new Y.Text(text); + files.set(fileName, ytext); + } } return confirmed; }; + private openProjectPlaintext = async (project: { + files: Record; + projectName: string; + }): Promise => { + await this.fs.replaceWithMultipleStrings(project); + }; + openIdea = async (slug: string | undefined, code: string, title: string) => { this.logging.event({ type: "idea-open", @@ -400,7 +435,9 @@ export class ProjectActions { { id: "confirm-replace-with-idea" }, { ideaName: pythonProject.projectName } ); - if (await this.openProject(pythonProject, confirmPrompt)) { + const { ydoc, id } = await this.newStoredProject(); + this.setStoredProjectName(id, pythonProject.projectName || "Untitled Idea"); + if (await this.openProject(pythonProject, ydoc, confirmPrompt)) { this.actionFeedback.success({ title: this.intl.formatMessage( { id: "loaded-file-feedback" }, @@ -410,14 +447,41 @@ export class ProjectActions { } }; + newProject = async () => { + const { ydoc, id } = await this.newStoredProject(); + this.setStoredProjectName(id, "Untitled project"); + await this.openProject(defaultInitialProject, ydoc); + }; + + loadProject = async (projectId: string) => { + const { projectName, ydoc } = await this.restoreStoredProject(projectId); + let files = {} as Record; + for (const filename of ydoc.getMap("files").keys()) { + const contents = ( + ydoc.getMap("files").get(filename) as Y.Text + ).toString(); + files[filename] = contents; + } + this.openProjectPlaintext({ + files, + projectName, + }); + }; + reset = async () => { + if (!this.ydoc) { + throw new Error("Attempted to reset a project with no storage backing"); + } this.logging.event({ type: "reset-project", }); const confirmPrompt = this.intl.formatMessage({ id: "confirm-replace-reset", }); - if (await this.openProject(defaultInitialProject, confirmPrompt)) { + + if ( + await this.openProject(defaultInitialProject, this.ydoc, confirmPrompt) + ) { this.actionFeedback.success({ title: this.intl.formatMessage({ id: "reset-project-feedback" }), }); @@ -775,6 +839,9 @@ export class ProjectActions { )); if (name) { await this.setProjectName(name); + if (this.projectId) { + await this.setStoredProjectName(this.projectId, name); + } return true; } return false; @@ -789,7 +856,9 @@ export class ProjectActions { this.logging.event({ type: "set-project-name", }); - + if (this.projectId) { + this.setStoredProjectName(this.projectId, name); + } return this.fs.setProjectName(name); }; diff --git a/src/project/project-hooks.tsx b/src/project/project-hooks.tsx index 3d3449ad9..b86686e6a 100644 --- a/src/project/project-hooks.tsx +++ b/src/project/project-hooks.tsx @@ -22,6 +22,8 @@ import { useSessionSettings } from "../settings/session-settings"; import { useSettings } from "../settings/settings"; import { useSelection } from "../workbench/use-selection"; import { defaultedProject, ProjectActions } from "./project-actions"; +import { useProjectList } from "../project-persistence/project-list-hooks"; +import { usePersistentProject } from "../project-persistence/persistent-project-hooks"; /** * Hook exposing the main UI actions. @@ -37,6 +39,10 @@ export const useProjectActions = (): ProjectActions => { const client = useLanguageServerClient(); const [settings, setSettings] = useSettings(); const [sessionSettings, setSessionSettings] = useSessionSettings(); + const { projectId, ydoc } = usePersistentProject(); + + const { newStoredProject, restoreStoredProject, setProjectName } = + useProjectList(); const actions = useMemo( () => new ProjectActions( @@ -49,7 +55,12 @@ export const useProjectActions = (): ProjectActions => { { values: sessionSettings, setValues: setSessionSettings }, intl, logging, - client + client, + ydoc || null, + projectId || null, + newStoredProject, + restoreStoredProject, + setProjectName ), [ fs, @@ -64,6 +75,11 @@ export const useProjectActions = (): ProjectActions => { setSettings, sessionSettings, setSessionSettings, + projectId, + ydoc, + newStoredProject, + restoreStoredProject, + setProjectName, ] ); return actions; diff --git a/src/router-hooks.tsx b/src/router-hooks.tsx index 0ccf1bac9..18e4226fc 100644 --- a/src/router-hooks.tsx +++ b/src/router-hooks.tsx @@ -22,7 +22,7 @@ import { import { baseUrl } from "./base"; import { useLogging } from "./logging/logging-hooks"; -export type TabName = "api" | "ideas" | "reference" | "project"; +export type TabName = "root" | "api" | "ideas" | "reference" | "project"; /** * An anchor-like navigation used for scroll positions. @@ -60,6 +60,9 @@ const parse = (pathname: string): RouterState => { if (pathname) { const parts = pathname.split("/"); const tab = parts[0]; + if (tab === "") { + return { tab: "root" }; + } if ( tab === "api" || tab === "reference" ||