diff --git a/.gitignore b/.gitignore index 2b53fdcb..1f166293 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__ *.egg-info /dist /build +/_build /eggs /parts /bin diff --git a/Makefile b/Makefile index 77885d0d..e5bf5554 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,14 @@ + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -W +SPHINXBUILD ?= sphinx-build +SOURCEDIR = docs +BUILDDIR = _build + .PHONY: clean compile_translations coverage diff_cover docs dummy_translations \ extract_translations fake_translations help pii_check pull_translations push_translations \ - quality requirements selfcheck test test-all upgrade validate + quality requirements selfcheck test test-all upgrade validate check_docs serve_docs .DEFAULT_GOAL := help @@ -101,3 +109,15 @@ dummy_translations: ## generate dummy translation (.po) files build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files validate_translations: build_dummy_translations detect_changed_source_translations ## validate translations + +serve_docs: + sphinx-autobuild -W docs/ docs/_build/html/ + +# Emulate the build step on RTD to flush out errors ahead pushing +check_docs: + sphinx-build -T -E -W --keep-going docs/ docs/_build/html + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/Makefile b/docs/Makefile index e01e682a..23eff373 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,10 +1,10 @@ # Makefile for Sphinx documentation -# -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build # User-friendly check for sphinx-build @@ -228,3 +228,8 @@ dummy: $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy @echo @echo "Build finished. Dummy builder generates no files." + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 6d6f957d..de3c9d8c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,6 +70,8 @@ def get_version(*file_paths): 'sphinx.ext.autosectionlabel' ] +autosectionlabel_prefix_document = True + # A list of warning types to suppress arbitrary warning messages. suppress_warnings = [ 'image.nonlocal_uri', @@ -112,7 +114,7 @@ def get_version(*file_paths): # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -315,6 +317,16 @@ def get_version(*file_paths): # Output file base name for HTML help builder. htmlhelp_basename = f'{project}doc' +# -- Read the Docs Specific Configuration +# Define the canonical URL if you are using a custom domain on Read the Docs +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") + +# Tell Jinja2 templates the build is running on Read the Docs +if os.environ.get("READTHEDOCS", "") == "True": + if "html_context" not in globals(): + html_context = {} + html_context["READTHEDOCS"] = True + # -- Options for LaTeX output --------------------------------------------- latex_elements = { diff --git a/docs/event-mapping/Supported_events.rst b/docs/event-mapping/Supported_events.rst deleted file mode 100644 index 95794b77..00000000 --- a/docs/event-mapping/Supported_events.rst +++ /dev/null @@ -1,141 +0,0 @@ - -List of supported edx events -============================ - -edX events supported by ``event-routing-backends`` are listed below. - -Enrollment events ------------------ - -* `edx.course.enrollment.activated`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.enrollment.activated.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-course-enrollment-activated>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.enrollment.activated.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-course-enrollment-activated>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.course.enrollment.activated.json>`__ -* `edx.course.enrollment.deactivated`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.enrollment.deactivated.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-course-enrollment-deactivated>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.enrollment.deactivated.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-course-enrollment-deactivated>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.course.enrollment.deactivated.json>`__ -* `edx.course.enrollment.mode_changed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.enrollment.mode_changed.json>`__ | xAPI `map <./xAPI_mapping.rst#edxcourseenrollmentmode_changed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.enrollment.mode_changed.json>`__ - -Course grading events ------------------------ - -* `edx.course.grade.passed.first_time`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.grade.passed.first_time.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-course-grade-passed-first-time>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.grade.passed.first_time.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-course-grade-passed-first-time>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.course.grade.passed.first_time.json>`__ -* `edx.course.grade.now_passed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.grade.now_passed.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-course-grade-now-passed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.grade.now_passed.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-course-grade-now-passed>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.course.grade.now_passed.json>`__ -* `edx.course.grade.now_failed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.course.grade.now_failed.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-course-grade-now-failed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.course.grade.now_failed.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-course-grade-now-failed>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.course.grade.now_failed.json>`__ -* `edx.grades.subsection.grade_calculated`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.grades.subsection.grade_calculated.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-grades-subsection-grade-calculated>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.grades.subsection.grade_calculated.json>`__ -* `edx.grades.course.grade_calculated`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.grades.course.grade_calculated.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-grades-course-grade-calculated>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.grades.course.grade_calculated.json>`__ - -Completion events - -* `edx.completion.block_completion.changed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.completion.block_completion.changed.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-completion-block-completion-changed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.completion.block_completion.changed.json>`__ - -Problem interaction events ---------------------------- - -* `problem_check`_ with ``event_source`` as ``server`` | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/problem_check(server).json>`__ | xAPI `map <./xAPI_mapping.rst#problem-check-event-source-server>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/problem_check(server).json>`__ | Caliper `map <./Caliper_mapping.rst#problem-check-event-source-server>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/problem_check(server).json>`__ -* `problem_check`_ with ``event_source`` as ``browser`` | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/problem_check(browser).json>`__ | xAPI `map <./xAPI_mapping.rst#problem-check-event-source-browser>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/problem_check(browser).json>`__ | Caliper `map <./Caliper_mapping.rst#problem-check-event-source-browser>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/problem_check(browser).json>`__ -* `showanswer`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/showanswer.json>`__ | xAPI `map <./xAPI_mapping.rst#showanswer>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/showanswer.json>`__ | Caliper `map <./Caliper_mapping.rst#showanswer>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/showanswer.json>`__ -* `edx.problem.hint.demandhint_displayed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.problem.hint.demandhint_displayed.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-problem-hint-demandhint-displayed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.problem.hint.demandhint_displayed.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-problem-hint-demandhint-displayed>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.problem.hint.demandhint_displayed.json>`__ - -Video events -------------- - -* `edx.video.loaded`_ (legacy name: ``load_video``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/load_video.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-loaded>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/load_video.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-loaded>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/load_video.json>`__ -* `edx.video.played`_ (legacy name: ``play_video``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/play_video.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-played>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/play_video.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-played>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/play_video.json>`__ -* `edx.video.stopped`_ (legacy name: ``stop_video``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/stop_video.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-stopped>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/stop_video.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-stopped>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/stop_video.json>`__ -* `edx.video.paused`_ (legacy name: ``pause_video``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/pause_video.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-paused>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/pause_video.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-paused>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/pause_video.json>`__ -* `edx.video.position.changed`_ (legacy name: ``seek_video``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/seek_video.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-position-changed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/seek_video.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-position-changed>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/seek_video.json>`__ -* complete_video | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/complete_video.json>`__ | xAPI `map <./xAPI_mapping.rst#complete_video>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/complete_video.json>`__ | Caliper `map <./Caliper_mapping.rst#complete_video>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/complete_video.json>`__ -* `edx.video.closed_captions.shown`_ (legacy name: ``video_show_cc_menu``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/video_show_cc_menu.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-closed_captions-shown>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/video_show_cc_menu.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-closed_captions-shown>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/video_show_cc_menu.json>`__ -* `edx.video.closed_captions.hidden`_ (legacy name: ``video_hide_cc_menu``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/video_hide_cc_menu.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-closed_captions-hidden>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/video_hide_cc_menu.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-closed_captions-hidden>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/video_hide_cc_menu.json>`__ -* `edx.video.transcript.shown`_ (legacy name: ``show_transcript``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/show_transcript.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-transcript-shown>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/show_transcript.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-transcript-shown>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/show_transcript.json>`__ -* `edx.video.transcript.hidden`_ (legacy name: ``hide_transcript``) | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/hide_transcript.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-video-transcript-hidden>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/hide_transcript.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-video-transcript-hidden>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/hide_transcript.json>`__ -* `speed_change_video`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/speed_change_video.json>`__ | xAPI `map <./xAPI_mapping.rst#speed_change_video>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/speed_change_video.json>`__ | Caliper `map <./Caliper_mapping.rst#speed_change_video>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/speed_change_video.json>`__ - -Course navigation events ------------------------- - -* `edx.ui.lms.sequence.outline.selected`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.ui.lms.outline.selected.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-ui-lms-sequence-outline-selected>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.ui.lms.outline.selected.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-ui-lms-sequence-outline-selected>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.ui.lms.outline.selected.json>`__ -* `edx.ui.lms.sequence.next_selected`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.ui.lms.sequence.next_selected.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-ui-lms-sequence-next-selected>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.ui.lms.sequence.next_selected.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-ui-lms-sequence-next-selected>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.ui.lms.sequence.next_selected.json>`__ -* `edx.ui.lms.sequence.previous_selected`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.ui.lms.sequence.previous_selected.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-ui-lms-sequence-previous-selected>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.ui.lms.sequence.previous_selected.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-ui-lms-sequence-previous-selected>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.ui.lms.sequence.previous_selected.json>`__ -* `edx.ui.lms.sequence.tab_selected`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.ui.lms.sequence.tab_selected.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-ui-lms-sequence-tab-selected>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.ui.lms.sequence.tab_selected.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-ui-lms-sequence-tab-selected>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.ui.lms.sequence.tab_selected.json>`__ -* `edx.ui.lms.link_clicked`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.ui.lms.link_clicked.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-ui-lms-link-clicked>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.ui.lms.link_clicked.json>`__ | Caliper `map <./Caliper_mapping.rst#edx-ui-lms-link-clicked>`__ , `sample <../../event_routing_backends/processors/caliper/tests/fixtures/expected/edx.ui.lms.link_clicked.json>`__ - -Forum events ------------------ - -* `edx.forum.thread.created`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.created.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-created>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.created.json>`__ -* `edx.forum.thread.edited`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.edited.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-edited>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.edited.json>`__ -* `edx.forum.thread.viewed`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.viewed.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-viewed>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.viewed.json>`__ -* `edx.forum.thread.deleted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.deleted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-deleted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.deleted.json>`__ -* `edx.forum.thread.voted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.voted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-voted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.voted.json>`__ -* `edx.forum.thread.reported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.reported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-reported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.reported.json>`__ -* `edx.forum.thread.unreported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.thread.unreported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-forum-thread-unreported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.thread.unreported.json>`__ -* `edx.forum.response.created`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.created.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.created>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.created.json>`__ -* `edx.forum.response.edited`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.edited.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.edited>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.edited.json>`__ -* `edx.forum.response.deleted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.deleted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.deleted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.deleted.json>`__ -* `edx.forum.response.voted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.voted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.voted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.voted.json>`__ -* `edx.forum.response.reported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.reported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.reported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.reported.json>`__ -* `edx.forum.response.unreported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.response.unreported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.response.unreported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.response.unreported.json>`__ -* `edx.forum.comment.created`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.comment.created.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.comment.created>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.comment.created.json>`__ -* `edx.forum.comment.edited`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.comment.edited.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.comment.edited>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.comment.edited.json>`__ -* `edx.forum.comment.deleted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.comment.deleted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.comment.deleted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.comment.deleted.json>`__ -* `edx.forum.comment.reported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.comment.reported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.comment.reported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.comment.reported.json>`__ -* `edx.forum.comment.unreported`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.forum.comment.unreported.json>`__ | xAPI `map <./xAPI_mapping.rst#edx.forum.comment.unreported>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.forum.comment.unreported.json>`__ - -Exam events ------------------- - -* `edx.special_exam.timed.attempt.created`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.special_exam.timed.attempt.started.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-special-exam-timed-attempt-started>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.special_exam.timed.attempt.started.json>`__ -* `edx.special_exam.timed.attempt.submitted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.special_exam.timed.attempt.submitted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-special-exam-timed-attempt-submitted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.special_exam.timed.attempt.submitted.json>`__ -* `edx.special_exam.proctored.attempt.created`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.special_exam.proctored.attempt.started.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-special-exam-proctored-attempt-started>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.special_exam.proctored.attempt.started.json>`__ -* `edx.special_exam.proctored.attempt.submitted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.special_exam.proctored.attempt.submitted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-special-exam-proctored-attempt-submitted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.special_exam.proctored.attempt.submitted.json>`__ -* `edx.special_exam.practice.attempt.submitted`_ | edX `sample <../../event_routing_backends/processors/tests/fixtures/current/edx.special_exam.practice.attempt.submitted.json>`__ | xAPI `map <./xAPI_mapping.rst#edx-special-exam-practice-attempt-submitted>`__ , `sample <../../event_routing_backends/processors/xapi/tests/fixtures/expected/edx.special_exam.practice.attempt.submitted.json>`__ - -.. _edx.course.enrollment.activated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-activated-and-edx-course-enrollment-deactivated -.. _edx.course.enrollment.deactivated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-activated-and-edx-course-enrollment-deactivated -.. _edx.course.enrollment.mode_changed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-mode-changed -.. _edx.course.grade.passed.first_time: https://github.com/openedx/docs.openedx.org/issues/855 -.. _edx.grades.subsection.grade_calculated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-subsection-grade-calculated -.. _edx.grades.course.grade_calculated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-course-grade-calculated -.. _edx.completion.block_completion.changed: https://github.com/openedx/docs.openedx.org/issues/855 -.. _edx.course.grade.now_passed: https://github.com/openedx/docs.openedx.org/issues/855 -.. _edx.course.grade.now_failed: https://github.com/openedx/docs.openedx.org/issues/855 -.. _edx.grades.problem.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-problem-submitted -.. _problem_check: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#problem-check -.. _showanswer: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#showanswer -.. _edx.problem.hint.demandhint_displayed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-problem-hint-demandhint-displayed -.. _edx.video.loaded: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#load-video-edx-video-loaded -.. _edx.video.played: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#play-video-edx-video-played -.. _edx.video.stopped: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#stop-video-edx-video-stopped -.. _edx.video.paused: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#pause-video-edx-video-paused -.. _edx.video.position.changed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#seek-video-edx-video-position-changed -.. _edx.ui.lms.sequence.outline.selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-outline-selected -.. _edx.ui.lms.sequence.next_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-next-selected -.. _edx.ui.lms.sequence.previous_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-previous-selected -.. _edx.ui.lms.sequence.tab_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-tab-selected -.. _edx.ui.lms.link_clicked: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-link-clicked -.. _edx.video.closed_captions.shown: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#video-show-cc-menu-edx-video-language-menu-shown -.. _edx.video.closed_captions.hidden: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#video-hide-cc-menu-edx-video-language-menu-hidden -.. _edx.video.transcript.shown: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#show-transcript-edx-video-transcript-shown -.. _edx.video.transcript.hidden: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#hide-transcript-edx-video-transcript-hidden -.. _speed_change_video: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#speed-change-video -.. _edx.forum.thread.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-created -.. _edx.forum.thread.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-edited -.. _edx.forum.thread.viewed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-viewed -.. _edx.forum.thread.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-deleted -.. _edx.forum.thread.voted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-voted -.. _edx.forum.thread.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-reported -.. _edx.forum.thread.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-unreported -.. _edx.forum.response.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-created -.. _edx.forum.response.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-edited -.. _edx.forum.response.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-deleted -.. _edx.forum.response.voted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-voted -.. _edx.forum.response.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-reported -.. _edx.forum.response.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-unreported -.. _edx.forum.comment.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-created -.. _edx.forum.comment.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-edited -.. _edx.forum.comment.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-deleted -.. _edx.forum.comment.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-reported -.. _edx.forum.comment.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-unreported -.. _edx.special_exam.timed.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created -.. _edx.special_exam.timed.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted -.. _edx.special_exam.proctored.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created -.. _edx.special_exam.proctored.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted -.. _edx.special_exam.practice.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created -.. _edx.special_exam.practice.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted diff --git a/docs/getting_started.rst b/docs/getting_started.rst deleted file mode 100644 index e6f66dfc..00000000 --- a/docs/getting_started.rst +++ /dev/null @@ -1,331 +0,0 @@ - -Introduction -=============== - -``event-routing-backends`` is developed as a pluggable application for the edx-platform. The code in this app hooks into the `event-tracking`_ app that is installed as a part of edx-platform. It provides new tracking backends and processors. - -Features --------- - -Events that need to be transformed can be filtered by their names using either `RegexFilter`_ processor or `NameWhitelist`_ processor offered by the ``event-tracking`` library. Both these processors run in the main thread. ``NameWhitelist`` performs simple string comparisons and is therefore, faster. - -In ``event_tracking_backends``, two processors, namely `CaliperProcessor`_ and `XApiProcessor`_ can transform edX events into Caliper and xAPI format respectively. Events in Caliper format need to be passed through an additional processor named `CaliperEnvelopeProcessor`_, after being transformed and before being routed. - -`EventsRouter`_ backend runs these processors and then routes the transformed events (xAPI or Caliper format) to configured routers. It is configured as a nested backend (named ``xapi`` or ``caliper``) of `AsyncRoutingBackend`_ (along with desired processors) in ``EVENT_TRACKING_BACKENDS`` configuration of the ``event-tracking`` library. - -Execution ---------- - -``RegexFilter`` and ``NameWhitelist`` run synchronously in the main thread. Processors in ``xapi`` and ``caliper`` backends are executed asynchronously, with each backend executed as a separate celery task. - -Routing of transformed events is also done asynchronously. Therefore, nested celery tasks are created, one for each configured router, to route events that have been transformed by ``xapi`` or ``caliper`` backends. - -Retries -------- - -Once an event fails to transmit due to connection error, it is retried periodically for a finite number of times, with delay between each retry. Total number of retries and delay (in seconds) between each retry can be configured using plugin setting ``EVENT_ROUTING_BACKEND_MAX_RETRIES`` (default: 3) and ``EVENT_ROUTING_BACKEND_COUNTDOWN`` (default: 30), respectively. If it still fails to transmit, then the event is dropped unless it is configured to persist in the database. - -Persistence ------------ - -Event consumers may never want to lose certain events even after a brief failure of the connection or at the endpoint. List of these events can be specified in plugin setting ``EVENT_TRACKING_BACKENDS_BUSINESS_CRITICAL_EVENTS``. Failed celery tasks for routing these events are persisted using `edx-celeryutils`_ package, once the retries have expired. ``edx-celeryutils`` also has `commands`_ for rerunning failed tasks and deleting old ones. Default list of events in ``EVENT_TRACKING_BACKENDS_BUSINESS_CRITICAL_EVENTS`` is: - -#. ``edx.course.enrollment.activated`` -#. ``edx.course.enrollment.deactivated`` -#. ``edx.course.grade.passed.first_time`` - -Supported events and mapping of edx events onto xAPI and Caliper formats ------------------------------------------------------------------------- - -List of supported edx events can be found in `Supported_events `_ along with their mapping onto xAPI and Caliper format. - -Version information of transformer ----------------------------------- - -Version of transformer is semantic version of event-routing-backend prefixed with `event-routing-backends@` included in the statement/event. Version is a string of format "event-routing-backend@X.Y.Z" where increment in X represents breaking changes and increment in Y represents addition/update of fields in the event/statement and Z represents bug fix or patched version. - -In xAPI statement, version is in value of the key ``https://w3id.org/xapi/openedx/extension/transformer-version`` in ``extensions`` of ``Context`` of the statement. - -In Caliper event, version is in value of the key ``transformerVersion`` in ``extensions`` of the event. - -Installation -=============== - -Install event routing backends library or add it to private requirements of your virtual environment ( ``requirements/private.txt`` ). - -#. Run ``pip install edx-event-routing-backends``. - -#. Run migrations ( ``python manage.py lms migrate`` ). - -#. Restart LMS service and celery workers of edx-platform. - -Configuration -=============== - -Two types of configuration are needed for the plugin: - -#. Routers for routing transformed events to desired http endpoints. - -#. Backends for filtering and transformation of selected events into ``xapi`` or ``caliper`` format. - -By default, both ``xapi`` and ``caliper`` backends are already configured along with filters that allow all the supported events. ``caliper`` backend is disabled by default and can be enabled by setting ``CALIPER_EVENTS_ENABLED`` to ``True`` in plugin settings. - -Additionally separate log streams for xAPI and Caliper are generated as events are transformed and can be configured to be saved or ignored. These can be configured as described in the `Django docs `_ for the ``xapi_tracking`` and ``caliper_tracking`` loggers. - - - -Router configuration --------------------- - -Router(s) for each backend can be configured in django admin settings as follows: - -#. Navigate to http://localhost:18000/admin/event_routing_backends/routerconfiguration/add/ - -#. Select ``Backend name`` (``xapi`` or ``caliper``). - -#. Add ``Route URL``; the HTTP endpoint where events are to be received. - -#. Select ``Auth Scheme`` (``Basic`` or ``Bearer`` or ``None``). For ``Basic`` authentication, add ``username`` and ``password``. For ``Bearer`` authentication, add ``Token``. - -#. Add ``Configurations`` comprising of following configuration items as json: - - #. ``override_args``: Accepts set of key:value pairs that will be added at the root level of the json of the event being routed. If the any of the keys already exist at the root level, their value will be overridden. Please note that for ``caliper`` backend, these changes will be made in the envelope. - - #. ``match_params``: This can be used to filter events based on values of keys in the original edX events. Regular expressions can be used for values. - - #. ``headers``: Additional headers can be specified here for ``caliper`` backend only. - -A sample configuration for routing Caliper events having content organisation as ``edX`` AND course run is 2021 AND event name starts with ``problem`` OR event name contains ``video``, with override arguments and additional headers: - -.. code-block:: JSON - - { - "override_args":{ - "sensor_id":"sensor@example.com" - }, - "headers":{ - "test":"header" - }, - "match_params":{ - "course_id":"^.*course-v.:edX\\+.*\\+2021.*$", - "name":[ - "^problem.*", - "video" - ] - } - } - -A sample configuration for routing xAPI events if the enterprise is ``org_XYZ`` AND event name is ``edx.course.grade.passed.first_time`` OR ``edx.course.enrollment.activated``: - -.. code-block:: JSON - - { - "match_params":{ - "enterprise_uuid":"org_XYZ", - "name":[ - "edx.course.grade.passed.first_time", - "edx.course.enrollment.activated" - ] - } - } - -Backends configuration ----------------------- - -By default, both ``caliper`` and ``xapi`` backends are configured with ``NameWhitelistProcessor`` that filters all the events currently supported. Users can override default backends to change filter type and name of the events to be filtered. - -A sample override for ``caliper`` backend is presented below. Here we are allowing only enrollment, ``seek_video`` and ``edx.video.position.changed`` events to be filtered through `RegexFilter`_ to ``caliper`` backend. - - .. code-block:: python - - EVENT_TRACKING_BACKENDS.update({ - 'caliper': { - 'ENGINE': 'eventtracking.backends.async_routing.AsyncRoutingBackend', - 'OPTIONS': { - 'backend_name': 'caliper', - 'processors': [ - { - 'ENGINE': 'eventtracking.processors.regex_filter.RegexFilter', - 'OPTIONS': { - 'filter_type': 'allowlist', - 'regular_expressions': [ - 'edx.course.enrollment.*', - 'seek_video', - 'edx.video.position.changed' - ] - } - } - ], - 'backends': { - 'caliper': { - 'ENGINE': 'event_routing_backends.backends.events_router.EventsRouter', - 'OPTIONS': { - 'processors': [ - { - 'ENGINE': 'event_routing_backends.processors.caliper.transformer_processor.CaliperProcessor', - 'OPTIONS': {} - }, - { - 'ENGINE': 'event_routing_backends.processors.caliper.envelope_processor.CaliperEnvelopeProcessor', - 'OPTIONS': { - 'sensor_id': 'http://example.com/sensors' - } - } - ], - 'backend_name': 'caliper' - } - } - } - } - } - }) - -A sample override for ``xapi`` backend is presented below. Here we are allowing only enrollment, ``edx.course.grade.passed.first_time`` and ``edx.ui.lms.sequence.tab_selected`` events to be filtered through `NameWhitelist`_ to ``xapi`` backend. - - .. code-block:: python - - EVENT_TRACKING_BACKENDS.update({ - 'xapi': { - 'ENGINE': 'eventtracking.backends.async_routing.AsyncRoutingBackend', - 'OPTIONS': { - 'backend_name': 'xapi', - 'processors': [ - { - 'ENGINE': 'eventtracking.processors.whitelist.NameWhitelistProcessor', - 'OPTIONS': { - 'whitelist': [ - 'edx.course.enrollment.activated', - 'edx.course.enrollment.deactivated', - 'edx.course.grade.passed.first_time', - 'edx.ui.lms.sequence.tab_selected', - ] - } - } - ], - 'backends': { - 'xapi': { - 'ENGINE': 'event_routing_backends.backends.events_router.EventsRouter', - 'OPTIONS': { - 'processors': [ - { - 'ENGINE': 'event_routing_backends.processors.xapi.transformer_processor.XApiProcessor', - 'OPTIONS': {} - } - ], - 'backend_name': 'xapi' - } - } - } - } - } - } - -Batching Configuration ----------------------- - -Batching of events can be configured using the following settings: - -#. ``EVENT_ROUTING_BACKEND_BATCHING_ENABLED``: If set to ``True``, events will be batched before being routed. Default is ``False``. -#. ``EVENT_ROUTING_BACKEND_BATCH_SIZE``: Maximum number of events to be batched together. Default is 100. -#. ``EVENT_ROUTING_BACKEND_BATCH_INTERVAL``: Time interval (in seconds) after which events will be ent, whether or not the batch size criteria is met. Default is 60 seconds. - -Batching is done in the ``EventsRouter`` backend. If ``EVENT_ROUTING_BACKEND_BATCHING_ENABLED`` is set to ``True``, then events will be batched together and routed to the configured routers after the specified interval or when the batch size is reached, whichever happens first. - -In case of downtimes or network issues, events will be queued again to avoid data loss. However, there is no guarantee that the events will be routed in the same order as they were received. - -Event bus configuration ------------------------ - -The event bus backend can be configured as the producer of the events. In that case, the events will be consumed from the event bus and routed to the configured routers via event bus consumers. The event bus backend can be configured in your edx-platform settings as follows: - -.. code-block:: python - - EVENT_TRACKING_BACKENDS["xapi"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" - EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"]["backends"]["xapi"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" - EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"].pop("backend_name") - if "openedx_events" not in INSTALLED_APPS: - INSTALLED_APPS.append("openedx_events") - SEND_TRACKING_EVENT_EMITTED_SIGNAL = True - EVENT_BUS_PRODUCER_CONFIG = { - "org.openedx.analytics.tracking.event.emitted.v1": { - "analytics": { - "event_key_field": "tracking_log.name", "enabled": True - } - } - } - -Once the event bus producer has been configured, the event bus consumer can be started using the following command: - -.. code-block:: bash - - ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "event_routing_backends"}' - -OpenEdx Filters -=============== - -This is an integration that allows to modify current standard outputs by using the `openedx-filters`_ library and is limited to the following filters per processor: - - -xAPI Filters ------------- - -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| Filter | Description | -+=================================================================================================+====================================================================================+ -| event_routing_backends.processors.xapi.transformer.xapi_transformer.get_actor | Intercepts and allows to modify the xAPI actor field, this affects all xAPI events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.transformer.xapi_transformer.get_verb | Intercepts and allows to modify the xAPI actor field, this affects all xAPI events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.completion_events.completion_created.get_object | Allows to modify the xAPI object field, this just affects completion events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.enrollment_events.base_enrollment.get_object | Allows to modify the xAPI object field, this just affects enrollment events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.exam_events.base_exam.get_object | Allows to modify the xAPI object field, this just affects exam events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.forum_events.base_forum_thread.get_object | Allows to modify the xAPI object field, this just affects forum events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.grading_events.subsection_graded.get_object | Allows to modify the xAPI object field, this just affects subsection_graded events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.grading_events.course_graded.get_object | Allows to modify the xAPI object field, this just affects course_graded events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.navigation_events.link_clicked.get_object | Allows to modify the xAPI object field, this just affects link_clicked events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.navigation_events.outline_selected.get_object | Allows to modify the xAPI object field, this just affects outline_selected events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.navigation_events.tab_navigation.get_object | Allows to modify the xAPI object field, this just affects tab_navigation events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.problem_interaction_events.base_problems.get_object | Allows to modify the xAPI object field, this just affects problem events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.problem_interaction_events.base_problem_check.get_object | Allows to modify the xAPI object field, this just affects problem_check events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -| event_routing_backends.processors.xapi.video_events.base_video.get_object | Allows to modify the xAPI object field, this affects all video events | -+-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ - -.. _event-tracking: https://github.com/openedx/event-tracking - -.. _NameWhitelist: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/whitelist.py - -.. _RegexFilter: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/regex_filter.py - -.. _save_statement: https://github.com/openedx/event-routing-backends/blob/2ec15d054b3b1dd6072689aa470f3d805486526e/event_routing_backends/utils/xapi_lrs_client.py#L70 - -.. _post: https://github.com/openedx/event-routing-backends/blob/2ec15d054b3b1dd6072689aa470f3d805486526e/event_routing_backends/utils/http_client.py#L67 - -.. _AsyncRoutingBackend: https://github.com/openedx/event-tracking/blob/fccad3d118f594fe304ec48517e896447f15e782/eventtracking/backends/async_routing.py#L13 - -.. _CaliperProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/caliper/transformer_processor.py#L16 - -.. _XApiProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/xapi/transformer_processor.py#L16 - -.. _CaliperEnvelopeProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/caliper/envelope_processor.py#L12 - -.. _EventsRouter: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/backends/events_router.py#L15 - -.. _business_critical_events: https://github.com/openedx/event-routing-backends/blob/e375674156b347be833ad8c2479be2c4ff4b073f/event_routing_backends/helpers.py#L197 - -.. _edx-celeryutils: https://github.com/openedx/edx-celeryutils - -.. _commands: https://github.com/openedx/edx-celeryutils/tree/master/celery_utils/management/commands - -.. _openedx-filters: https://github.com/openedx/openedx-filters - diff --git a/docs/index.rst b/docs/index.rst index 7bcc2391..2809284d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,18 +1,89 @@ -.. event-routing-backends documentation top level file, created by - sphinx-quickstart on Tue Sep 22 16:30:25 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. _Event Routing Home: -event-routing-backends -====================== +Event Routing Backends +###################### -Various backends for receiving edX LMS events. +.. contents:: :local: + +Introduction +------------ + +``event-routing-backends`` is developed as a pluggable application for the edx-platform. The code in this app hooks into the `event-tracking`_ app that is installed as a part of edx-platform. It provides new tracking backends and processors. + +Features +-------- + +Events that need to be transformed can be filtered by their names using either `RegexFilter`_ processor or `NameWhitelist`_ processor offered by the ``event-tracking`` library. Both these processors run in the main thread. ``NameWhitelist`` performs simple string comparisons and is therefore, faster. + +In ``event_tracking_backends``, two processors, namely `CaliperProcessor`_ and `XApiProcessor`_ can transform edX events into Caliper and xAPI format respectively. Events in Caliper format need to be passed through an additional processor named `CaliperEnvelopeProcessor`_, after being transformed and before being routed. + +`EventsRouter`_ backend runs these processors and then routes the transformed events (xAPI or Caliper format) to configured routers. It is configured as a nested backend (named ``xapi`` or ``caliper``) of `AsyncRoutingBackend`_ (along with desired processors) in ``EVENT_TRACKING_BACKENDS`` configuration of the ``event-tracking`` library. + +Execution +--------- + +``RegexFilter`` and ``NameWhitelist`` run synchronously in the main thread. Processors in ``xapi`` and ``caliper`` backends are executed asynchronously, with each backend executed as a separate celery task. + +Routing of transformed events is also done asynchronously. Therefore, nested celery tasks are created, one for each configured router, to route events that have been transformed by ``xapi`` or ``caliper`` backends. + +Retries +------- + +Once an event fails to transmit due to connection error, it is retried periodically for a finite number of times, with delay between each retry. Total number of retries and delay (in seconds) between each retry can be configured using plugin setting ``EVENT_ROUTING_BACKEND_MAX_RETRIES`` (default: 3) and ``EVENT_ROUTING_BACKEND_COUNTDOWN`` (default: 30), respectively. If it still fails to transmit, then the event is dropped unless it is configured to persist in the database. + +Persistence +----------- + +Event consumers may never want to lose certain events even after a brief failure of the connection or at the endpoint. List of these events can be specified in plugin setting ``EVENT_TRACKING_BACKENDS_BUSINESS_CRITICAL_EVENTS``. Failed celery tasks for routing these events are persisted using `edx-celeryutils`_ package, once the retries have expired. ``edx-celeryutils`` also has `commands`_ for rerunning failed tasks and deleting old ones. Default list of events in ``EVENT_TRACKING_BACKENDS_BUSINESS_CRITICAL_EVENTS`` is: + +#. ``edx.course.enrollment.activated`` +#. ``edx.course.enrollment.deactivated`` +#. ``edx.course.grade.passed.first_time`` + +Supported events +---------------- + +List of supported edx events and mapping of edx events onto xAPI and Caliper formats can be found in :ref:`supported_events`. + +Version information of transformer +---------------------------------- + +Version of transformer is semantic version of event-routing-backend prefixed with `event-routing-backends@` included in the statement/event. Version is a string of format "event-routing-backend@X.Y.Z" where increment in X represents breaking changes and increment in Y represents addition/update of fields in the event/statement and Z represents bug fix or patched version. + +In xAPI statement, version is in value of the key ``https://w3id.org/xapi/openedx/extension/transformer-version`` in ``extensions`` of ``Context`` of the statement. + +In Caliper event, version is in value of the key ``transformerVersion`` in ``extensions`` of the event. -Contents: .. toctree:: - :maxdepth: 2 + :hidden: + + Technial Documentation + + +.. _event-tracking: https://github.com/openedx/event-tracking + +.. _NameWhitelist: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/whitelist.py + +.. _RegexFilter: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/regex_filter.py + +.. _save_statement: https://github.com/openedx/event-routing-backends/blob/2ec15d054b3b1dd6072689aa470f3d805486526e/event_routing_backends/utils/xapi_lrs_client.py#L70 + +.. _post: https://github.com/openedx/event-routing-backends/blob/2ec15d054b3b1dd6072689aa470f3d805486526e/event_routing_backends/utils/http_client.py#L67 + +.. _AsyncRoutingBackend: https://github.com/openedx/event-tracking/blob/fccad3d118f594fe304ec48517e896447f15e782/eventtracking/backends/async_routing.py#L13 + +.. _CaliperProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/caliper/transformer_processor.py#L16 + +.. _XApiProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/xapi/transformer_processor.py#L16 + +.. _CaliperEnvelopeProcessor: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/processors/caliper/envelope_processor.py#L12 + +.. _EventsRouter: https://github.com/openedx/event-routing-backends/blob/ac192ab6b4d1452ada37302d1481eea2f58aef19/event_routing_backends/backends/events_router.py#L15 + +.. _business_critical_events: https://github.com/openedx/event-routing-backends/blob/e375674156b347be833ad8c2479be2c4ff4b073f/event_routing_backends/helpers.py#L197 + +.. _edx-celeryutils: https://github.com/openedx/edx-celeryutils + +.. _commands: https://github.com/openedx/edx-celeryutils/tree/master/celery_utils/management/commands - readme - getting_started - testing diff --git a/docs/readme.rst b/docs/readme.rst deleted file mode 100644 index 72a33558..00000000 --- a/docs/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../README.rst diff --git a/docs/event-mapping/Caliper_mapping.rst b/docs/technical_documentation/concepts/event-mapping/Caliper_mapping.rst similarity index 99% rename from docs/event-mapping/Caliper_mapping.rst rename to docs/technical_documentation/concepts/event-mapping/Caliper_mapping.rst index b9a72b31..cc877924 100644 --- a/docs/event-mapping/Caliper_mapping.rst +++ b/docs/technical_documentation/concepts/event-mapping/Caliper_mapping.rst @@ -1,3 +1,5 @@ +Caliper Mapping +############### edx.course.enrollment.activated =============================== @@ -52,7 +54,7 @@ type CourseOffering name =========== ============================================ -problem_check event_source_server +problem_check.event_source.server ==================================== ================================ ==================================================================================================== @@ -82,7 +84,7 @@ score [ attempt ] < data [ attempts ] > score [ extensions [ success ] ] TRUE if < data [success] > == "correct" else FALSE ================================ ==================================================================================================== -problem_check event_source_browser +problem_check.event_source.browser ===================================== =============== ============================================ diff --git a/docs/technical_documentation/concepts/event-mapping/Supported_events.rst b/docs/technical_documentation/concepts/event-mapping/Supported_events.rst new file mode 100644 index 00000000..c054df22 --- /dev/null +++ b/docs/technical_documentation/concepts/event-mapping/Supported_events.rst @@ -0,0 +1,143 @@ +.. _supported_events: + +Supported Events +################ + +edX events supported by ``event-routing-backends`` are listed below. + +Enrollment events +----------------- +* `edx.course.enrollment.activated`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.course.enrollment.deactivated`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.course.enrollment.mode_changed`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ + +Course grading events +----------------------- + +* `edx.course.grade.passed.first_time`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.course.grade.now_passed`_ | edX `sample `__ | xAPI `sample `__ +* `edx.course.grade.now_failed`_ | edX `sample `__ | xAPI `sample `__ | Caliper `sample `__ +* `edx.grades.subsection.grade_calculated`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.grades.course.grade_calculated`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ + +Completion events +----------------- + +* `edx.completion.block_completion.changed`_ | edX `sample `__ | xAPI `sample `__ + +Problem interaction events +--------------------------- + +* `problem_check`_ with ``event_source`` as ``server`` | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `problem_check`_ with ``event_source`` as ``browser`` | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `showanswer`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.problem.hint.demandhint_displayed`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.grades.problem.submitted`_ | edX `sample `__ | xAPI `sample `__ + +Video events +------------- + +* `edx.video.loaded`_ (legacy name: ``load_video``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.played`_ (legacy name: ``play_video``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.stopped`_ (legacy name: ``stop_video``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.paused`_ (legacy name: ``pause_video``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.position.changed`_ (legacy name: ``seek_video``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* complete_video | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.closed_captions.shown`_ (legacy name: ``video_show_cc_menu``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.closed_captions.hidden`_ (legacy name: ``video_hide_cc_menu``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.transcript.shown`_ (legacy name: ``show_transcript``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.video.transcript.hidden`_ (legacy name: ``hide_transcript``) | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `speed_change_video`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ + +Course navigation events +------------------------ + +* `edx.ui.lms.sequence.outline.selected`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.ui.lms.sequence.next_selected`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.ui.lms.sequence.previous_selected`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.ui.lms.sequence.tab_selected`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ +* `edx.ui.lms.link_clicked`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ | Caliper :ref:`map `, `sample `__ + +Forum events +----------------- + +* `edx.forum.thread.created`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.edited`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.viewed`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.deleted`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.voted`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.reported`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.thread.unreported`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.created`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.edited`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.deleted`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.voted`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.reported`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.response.unreported`_ | edX `sample `__ | xAPI :ref:`map `, `sample `__ +* `edx.forum.comment.created`_ | edX `sample `__ | xAPI `sample `__ +* `edx.forum.comment.edited`_ | edX `sample `__ | xAPI `sample `__ +* `edx.forum.comment.deleted`_ | edX `sample `__ | xAPI `sample `__ +* `edx.forum.comment.reported`_ | edX `sample `__ | xAPI `sample `__ +* `edx.forum.comment.unreported`_ | edX `sample `__ | xAPI `sample `__ + +Exam events +------------------ + +* `edx.special_exam.timed.attempt.created`_ | edX `sample `__ | xAPI `sample `__ +* `edx.special_exam.timed.attempt.submitted`_ | edX `sample `__ | xAPI `sample `__ +* `edx.special_exam.proctored.attempt.created`_ | edX `sample `__ | xAPI `sample `__ +* `edx.special_exam.proctored.attempt.submitted`_ | edX `sample `__ | xAPI `sample `__ +* `edx.special_exam.practice.attempt.submitted`_ | edX `sample `__ | xAPI `sample `__ + +.. _edx.course.enrollment.activated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-activated-and-edx-course-enrollment-deactivated +.. _edx.course.enrollment.deactivated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-activated-and-edx-course-enrollment-deactivated +.. _edx.course.enrollment.mode_changed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-course-enrollment-mode-changed +.. _edx.course.grade.passed.first_time: https://github.com/openedx/docs.openedx.org/issues/855 +.. _edx.grades.subsection.grade_calculated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-subsection-grade-calculated +.. _edx.grades.course.grade_calculated: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-course-grade-calculated +.. _edx.completion.block_completion.changed: https://github.com/openedx/docs.openedx.org/issues/855 +.. _edx.course.grade.now_passed: https://github.com/openedx/docs.openedx.org/issues/855 +.. _edx.course.grade.now_failed: https://github.com/openedx/docs.openedx.org/issues/855 +.. _edx.grades.problem.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/course_team_event_types.html#edx-grades-problem-submitted +.. _problem_check: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#problem-check +.. _showanswer: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#showanswer +.. _edx.problem.hint.demandhint_displayed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-problem-hint-demandhint-displayed +.. _edx.video.loaded: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#load-video-edx-video-loaded +.. _edx.video.played: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#play-video-edx-video-played +.. _edx.video.stopped: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#stop-video-edx-video-stopped +.. _edx.video.paused: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#pause-video-edx-video-paused +.. _edx.video.position.changed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#seek-video-edx-video-position-changed +.. _edx.ui.lms.sequence.outline.selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-outline-selected +.. _edx.ui.lms.sequence.next_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-next-selected +.. _edx.ui.lms.sequence.previous_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-previous-selected +.. _edx.ui.lms.sequence.tab_selected: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-sequence-tab-selected +.. _edx.ui.lms.link_clicked: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-ui-lms-link-clicked +.. _edx.video.closed_captions.shown: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#video-show-cc-menu-edx-video-language-menu-shown +.. _edx.video.closed_captions.hidden: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#video-hide-cc-menu-edx-video-language-menu-hidden +.. _edx.video.transcript.shown: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#show-transcript-edx-video-transcript-shown +.. _edx.video.transcript.hidden: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#hide-transcript-edx-video-transcript-hidden +.. _speed_change_video: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#speed-change-video +.. _edx.forum.thread.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-created +.. _edx.forum.thread.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-edited +.. _edx.forum.thread.viewed: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-viewed +.. _edx.forum.thread.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-deleted +.. _edx.forum.thread.voted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-voted +.. _edx.forum.thread.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-reported +.. _edx.forum.thread.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-thread-unreported +.. _edx.forum.response.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-created +.. _edx.forum.response.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-edited +.. _edx.forum.response.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-deleted +.. _edx.forum.response.voted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-voted +.. _edx.forum.response.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-reported +.. _edx.forum.response.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-response-unreported +.. _edx.forum.comment.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-created +.. _edx.forum.comment.edited: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-edited +.. _edx.forum.comment.deleted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-deleted +.. _edx.forum.comment.reported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-reported +.. _edx.forum.comment.unreported: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-forum-comment-unreported +.. _edx.special_exam.timed.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created +.. _edx.special_exam.timed.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted +.. _edx.special_exam.proctored.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created +.. _edx.special_exam.proctored.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted +.. _edx.special_exam.practice.attempt.created: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-created-edx-special-exam-practice-attempt-created-and-edx-special-exam-timed-attempt-created +.. _edx.special_exam.practice.attempt.submitted: https://docs.openedx.org/en/latest/developers/references/internal_data_formats/tracking_logs/student_event_types.html#edx-special-exam-proctored-attempt-submitted-edx-special-exam-practice-attempt-submitted-and-edx-special-exam-timed-attempt-submitted diff --git a/docs/technical_documentation/concepts/event-mapping/index.rst b/docs/technical_documentation/concepts/event-mapping/index.rst new file mode 100644 index 00000000..25c047e4 --- /dev/null +++ b/docs/technical_documentation/concepts/event-mapping/index.rst @@ -0,0 +1,9 @@ +Event Mapping +############# + +.. toctree:: + :maxdepth: 1 + + caliper_mapping + supported_events + xapi_mapping diff --git a/docs/event-mapping/xAPI_mapping.rst b/docs/technical_documentation/concepts/event-mapping/xAPI_mapping.rst similarity index 99% rename from docs/event-mapping/xAPI_mapping.rst rename to docs/technical_documentation/concepts/event-mapping/xAPI_mapping.rst index 64246254..e9d67b45 100644 --- a/docs/event-mapping/xAPI_mapping.rst +++ b/docs/technical_documentation/concepts/event-mapping/xAPI_mapping.rst @@ -1,3 +1,5 @@ +xAPI Mapping +############ edx.course.enrollment.activated =============================== @@ -81,7 +83,7 @@ definition [ type ] http://adlnet.gov/expapi/activities/course definition [ name ][ en-US ] ============================ ========================================== -problem_check event_source_server +problem_check.event_source.server ===================================== ========================================================================== ====================================================================================================== @@ -137,7 +139,7 @@ symbolicresponse fill-in truefalseresponse true-false ====================== =============== -problem_check event_source_browser +problem_check.event_source.browser ===================================== ============================================================= ================================================================================================================= diff --git a/docs/technical_documentation/concepts/index.rst b/docs/technical_documentation/concepts/index.rst new file mode 100644 index 00000000..85066067 --- /dev/null +++ b/docs/technical_documentation/concepts/index.rst @@ -0,0 +1,10 @@ +Concepts +######## + +.. toctree:: + :maxdepth: 1 + + Event Mapping + xapi_extensions + internationalization + diff --git a/docs/internationalization.rst b/docs/technical_documentation/concepts/internationalization.rst similarity index 96% rename from docs/internationalization.rst rename to docs/technical_documentation/concepts/internationalization.rst index 95b98a15..a98339c9 100644 --- a/docs/internationalization.rst +++ b/docs/technical_documentation/concepts/internationalization.rst @@ -1,7 +1,6 @@ -.. _chapter-i18n: - Internationalization -==================== +#################### + All user-facing text content should be marked for translation. Even if this application is only run in English, our open source users may choose to use another language. Marking content for translation ensures our users have this choice. @@ -11,7 +10,7 @@ Follow the `internationalization coding guidelines`_ in the Open edX Developer's .. _internationalization coding guidelines: https://docs.openedx.org/en/latest/developers/references/developer_guide/internationalization/i18n.html Updating Translations -~~~~~~~~~~~~~~~~~~~~~ +--------------------- This project uses `Transifex`_ to translate content. After new features are developed the translation source files should be pushed to Transifex. Our translation community will translate the content, after which we can retrieve the translations. @@ -38,7 +37,7 @@ The `make` targets listed below can be used to push or pull translations. - Push source translation files to Transifex Fake Translations -~~~~~~~~~~~~~~~~~ +----------------- As you develop features it may be helpful to know which strings have been marked for translation, and which are not. Use the `fake_translations` make target for this purpose. This target will extract all strings marked for translation, generate fake translations in the Esperanto (eo) language directory, and compile the translations. diff --git a/docs/xapi-extensions/eventVersion.rst b/docs/technical_documentation/concepts/xapi_extensions.rst similarity index 97% rename from docs/xapi-extensions/eventVersion.rst rename to docs/technical_documentation/concepts/xapi_extensions.rst index 43c83aed..3a59d396 100644 --- a/docs/xapi-extensions/eventVersion.rst +++ b/docs/technical_documentation/concepts/xapi_extensions.rst @@ -1,3 +1,6 @@ +xAPI Extensions +############### + +-------------------------------------------------------------+--------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Extensions | Title | Description | +=====================================================================================================+====================+====================================================================================================================================================================================+ diff --git a/docs/decisions/0001-purpose-of-this-repo.rst b/docs/technical_documentation/decisions/0001-purpose-of-this-repo.rst similarity index 98% rename from docs/decisions/0001-purpose-of-this-repo.rst rename to docs/technical_documentation/decisions/0001-purpose-of-this-repo.rst index 4fc7eeb1..979f416b 100644 --- a/docs/decisions/0001-purpose-of-this-repo.rst +++ b/docs/technical_documentation/decisions/0001-purpose-of-this-repo.rst @@ -1,5 +1,5 @@ 1. Purpose of this Repo -======================= +####################### Status ------ diff --git a/docs/decisions/0002-binding-app-for-caliper.rst b/docs/technical_documentation/decisions/0002-binding-app-for-caliper.rst similarity index 94% rename from docs/decisions/0002-binding-app-for-caliper.rst rename to docs/technical_documentation/decisions/0002-binding-app-for-caliper.rst index ffa1238f..324c83c4 100644 --- a/docs/decisions/0002-binding-app-for-caliper.rst +++ b/docs/technical_documentation/decisions/0002-binding-app-for-caliper.rst @@ -1,5 +1,5 @@ -1. Binding app for Caliper -========================== +2. Binding app for Caliper +########################## Status ------ diff --git a/docs/decisions/0003-router-configurations-storage.rst b/docs/technical_documentation/decisions/0003-router-configurations-storage.rst similarity index 95% rename from docs/decisions/0003-router-configurations-storage.rst rename to docs/technical_documentation/decisions/0003-router-configurations-storage.rst index 6453351e..393066c6 100644 --- a/docs/decisions/0003-router-configurations-storage.rst +++ b/docs/technical_documentation/decisions/0003-router-configurations-storage.rst @@ -1,5 +1,5 @@ -1. Router Configurations Storage -================================ +3. Router Configurations Storage +################################ Status ------ diff --git a/docs/decisions/0004-transformers-architecture.md b/docs/technical_documentation/decisions/0004-transformers-architecture.rst similarity index 97% rename from docs/decisions/0004-transformers-architecture.md rename to docs/technical_documentation/decisions/0004-transformers-architecture.rst index a24bb789..e16aac33 100644 --- a/docs/decisions/0004-transformers-architecture.md +++ b/docs/technical_documentation/decisions/0004-transformers-architecture.rst @@ -1,5 +1,5 @@ 4. Transformers Architecture -============================ +############################ Status ------ diff --git a/docs/decisions/0005-PII-leakage-prevention.rst b/docs/technical_documentation/decisions/0005-PII-leakage-prevention.rst similarity index 96% rename from docs/decisions/0005-PII-leakage-prevention.rst rename to docs/technical_documentation/decisions/0005-PII-leakage-prevention.rst index ba773a46..284d1665 100644 --- a/docs/decisions/0005-PII-leakage-prevention.rst +++ b/docs/technical_documentation/decisions/0005-PII-leakage-prevention.rst @@ -1,5 +1,5 @@ -PII leakage prevention -================================ +5. PII leakage prevention +######################### Status ------ diff --git a/docs/decisions/0006-versioning-of-event-transformers.rst b/docs/technical_documentation/decisions/0006-versioning-of-event-transformers.rst similarity index 96% rename from docs/decisions/0006-versioning-of-event-transformers.rst rename to docs/technical_documentation/decisions/0006-versioning-of-event-transformers.rst index b3d21013..23683a91 100644 --- a/docs/decisions/0006-versioning-of-event-transformers.rst +++ b/docs/technical_documentation/decisions/0006-versioning-of-event-transformers.rst @@ -1,5 +1,5 @@ -Versioning of event transformers -================================ +6. Versioning of event transformers +################################### Status ------ diff --git a/docs/decisions/0007-access-control-of-events.rst b/docs/technical_documentation/decisions/0007-access-control-of-events.rst similarity index 88% rename from docs/decisions/0007-access-control-of-events.rst rename to docs/technical_documentation/decisions/0007-access-control-of-events.rst index 1ab8a63b..1a600c5b 100644 --- a/docs/decisions/0007-access-control-of-events.rst +++ b/docs/technical_documentation/decisions/0007-access-control-of-events.rst @@ -1,5 +1,5 @@ -Access control of events -======================== +7. Access control of events +########################### Status ------ @@ -9,7 +9,7 @@ Approved Context ------- -`event-routing-backends` can transform filtered edX events in xAPI and Caliper format which can then be routed to configured URLs. Procedure for configuring URLs of learning record consumers can be found in the :ref:`Installation`. List of supported edX events can be accessed `here`_. A method is required to filter edX events based on event name, course id, organisation id etc. or on any combination of these properties. +`event-routing-backends` can transform filtered edX events in xAPI and Caliper format which can then be routed to configured URLs. Procedure for configuring URLs of learning record consumers can be found in the :ref:`Installation`. List of supported edX events can be accessed :ref:`here `. A method is required to filter edX events based on event name, course id, organisation id etc. or on any combination of these properties. Decision -------- @@ -54,5 +54,3 @@ Examples "match_params": { "enterprise_uuid": "org_XYZ", "name": ["edx.course.completed", "edx.course.enrollment.activated"]} - -.. _here: ../event-mapping/Supported_events.rst diff --git a/docs/decisions/0009-persistence-and-retries-for-events.rst b/docs/technical_documentation/decisions/0009-persistence-and-retries-for-events.rst similarity index 96% rename from docs/decisions/0009-persistence-and-retries-for-events.rst rename to docs/technical_documentation/decisions/0009-persistence-and-retries-for-events.rst index da17d901..5996c66e 100644 --- a/docs/decisions/0009-persistence-and-retries-for-events.rst +++ b/docs/technical_documentation/decisions/0009-persistence-and-retries-for-events.rst @@ -1,5 +1,5 @@ -Persistence and retries for events -================================== +9. Persistence and retries for events +##################################### Status ------ diff --git a/docs/technical_documentation/decisions/index.rst b/docs/technical_documentation/decisions/index.rst new file mode 100644 index 00000000..b5a69fa9 --- /dev/null +++ b/docs/technical_documentation/decisions/index.rst @@ -0,0 +1,14 @@ +Decisions +######### + +.. toctree:: + :maxdepth: 1 + + 0001-purpose-of-this-repo + 0002-binding-app-for-caliper + 0003-router-configurations-storage + 0004-transformers-architecture + 0005-PII-leakage-prevention + 0006-versioning-of-event-transformers + 0007-access-control-of-events + 0009-persistence-and-retries-for-events diff --git a/docs/technical_documentation/how-tos/filters.rst b/docs/technical_documentation/how-tos/filters.rst new file mode 100644 index 00000000..4b64537f --- /dev/null +++ b/docs/technical_documentation/how-tos/filters.rst @@ -0,0 +1,44 @@ +.. _filters: + +OpenEdx Filters +############### + +This is an integration that allows to modify current standard outputs by using the `openedx-filters`_ library and is limited to the following filters per processor: + + +xAPI Filters +------------ + ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| Filter | Description | ++=================================================================================================+====================================================================================+ +| event_routing_backends.processors.xapi.transformer.xapi_transformer.get_actor | Intercepts and allows to modify the xAPI actor field, this affects all xAPI events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.transformer.xapi_transformer.get_verb | Intercepts and allows to modify the xAPI actor field, this affects all xAPI events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.completion_events.completion_created.get_object | Allows to modify the xAPI object field, this just affects completion events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.enrollment_events.base_enrollment.get_object | Allows to modify the xAPI object field, this just affects enrollment events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.exam_events.base_exam.get_object | Allows to modify the xAPI object field, this just affects exam events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.forum_events.base_forum_thread.get_object | Allows to modify the xAPI object field, this just affects forum events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.grading_events.subsection_graded.get_object | Allows to modify the xAPI object field, this just affects subsection_graded events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.grading_events.course_graded.get_object | Allows to modify the xAPI object field, this just affects course_graded events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.navigation_events.link_clicked.get_object | Allows to modify the xAPI object field, this just affects link_clicked events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.navigation_events.outline_selected.get_object | Allows to modify the xAPI object field, this just affects outline_selected events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.navigation_events.tab_navigation.get_object | Allows to modify the xAPI object field, this just affects tab_navigation events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.problem_interaction_events.base_problems.get_object | Allows to modify the xAPI object field, this just affects problem events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.problem_interaction_events.base_problem_check.get_object | Allows to modify the xAPI object field, this just affects problem_check events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| event_routing_backends.processors.xapi.video_events.base_video.get_object | Allows to modify the xAPI object field, this affects all video events | ++-------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ + +.. _openedx-filters: https://github.com/openedx/openedx-filters diff --git a/docs/howto/how_to_bulk_transform.rst b/docs/technical_documentation/how-tos/how_to_bulk_transform.rst similarity index 98% rename from docs/howto/how_to_bulk_transform.rst rename to docs/technical_documentation/how-tos/how_to_bulk_transform.rst index 1164054d..3fed8f12 100644 --- a/docs/howto/how_to_bulk_transform.rst +++ b/docs/technical_documentation/how-tos/how_to_bulk_transform.rst @@ -1,5 +1,5 @@ How To Bulk Transform Tracking Logs -=================================== +################################### This is a rough guide of how to transform existing tracking log files into the formats supported by event-routing-backends using the ``transform_tracking_logs`` Django management command inside a running LMS installation. Because the transformations perform database access, looking up user, course, and block data, you will need to run this command on the same install of Open edX that created the tracking log files. @@ -44,7 +44,7 @@ Additionally all generated statements are written to a Python logger which can b **File(s) to logger** - For any destination you can use the ``--dry_run`` flag to perform tests on finding and transforming data before attempting to store it. Used in conjunction with loggers mentioned above, you can use Python log forwarding without the additional overhead of storing full files. .. warning:: - Events may be filtered differently in this command than in normal operation. Normally events pass through two layers of filters as described in `getting started `_. + Events may be filtered differently in this command than in normal operation. Normally events pass through two layers of filters as described :ref:`here `. First are the eventtracking AsyncRoutingBackend can have processor filters, which will be ignored when running this script (since these events have already passed through the eventtracking process). diff --git a/docs/howto/how_to_test.rst b/docs/technical_documentation/how-tos/how_to_test.rst similarity index 85% rename from docs/howto/how_to_test.rst rename to docs/technical_documentation/how-tos/how_to_test.rst index ab87e70f..83cfb825 100644 --- a/docs/howto/how_to_test.rst +++ b/docs/technical_documentation/how-tos/how_to_test.rst @@ -1,9 +1,7 @@ -=========== How To Test -=========== - -This is a rough guide of how to go about testing backends in edx-platform. It assumes you have devstack already up and running. If not, see `devstack docs `_. +########### +This is a rough guide of how to go about testing backends in edx-platform. Add edx-event-routing-backends to requirements ---------------------------------------------- @@ -40,7 +38,7 @@ Set the Django setting ``CALIPER_EVENTS_ENABLED`` or ``XAPI_EVENTS_ENABLED`` to Configure for testing --------------------- -Follow instructions in `getting started `_ doc. There is a chance some of this has already been done. Most of the code changes should just be copy and paste, but here are some suggested changes for debugging. It'll be easier to understand following instructions if you read `getting_started `_ doc first. +Follow instructions in :ref:`configuration `. There is a chance some of this has already been done. Most of the code changes should just be copy and paste, but here are some suggested changes for debugging. It'll be easier to understand following instructions if you read `getting_started `_ doc first. Change router endpoint ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/technical_documentation/how-tos/index.rst b/docs/technical_documentation/how-tos/index.rst new file mode 100644 index 00000000..2ee5b48e --- /dev/null +++ b/docs/technical_documentation/how-tos/index.rst @@ -0,0 +1,9 @@ +How Tos +####### + +.. toctree:: + :maxdepth: 1 + + filters + how_to_bulk_transform + how_to_test diff --git a/docs/technical_documentation/index.rst b/docs/technical_documentation/index.rst new file mode 100644 index 00000000..a409c779 --- /dev/null +++ b/docs/technical_documentation/index.rst @@ -0,0 +1,14 @@ +.. _technical-documentation: + +Technical Documentation +####################### + +``event-routing-backends`` contains plugins for the event-tracking app that is installed as a part of ``edx-platform``. It provides a backend that can take events and re-transmit them to external services and provides some new processers that can convert edx-platform events into other formats. + +.. toctree:: + :maxdepth: 2 + + Concepts + How-To's + Quickstarts + Decisions diff --git a/docs/technical_documentation/quickstarts/configuration.rst b/docs/technical_documentation/quickstarts/configuration.rst new file mode 100644 index 00000000..d8ae3985 --- /dev/null +++ b/docs/technical_documentation/quickstarts/configuration.rst @@ -0,0 +1,206 @@ +.. _configuration: + +Configuration +############# + +Two types of configuration are needed for the plugin: + +#. Routers for routing transformed events to desired http endpoints. + +#. Backends for filtering and transformation of selected events into ``xapi`` or ``caliper`` format. + +By default, both ``xapi`` and ``caliper`` backends are already configured along with filters that allow all the supported events. ``caliper`` backend is disabled by default and can be enabled by setting ``CALIPER_EVENTS_ENABLED`` to ``True`` in plugin settings. + +Additionally separate log streams for xAPI and Caliper are generated as events are transformed and can be configured to be saved or ignored. These can be configured as described in the `Django docs `_ for the ``xapi_tracking`` and ``caliper_tracking`` loggers. + + + +Router configuration +-------------------- + +Router(s) for each backend can be configured in django admin settings as follows: + +#. Navigate to http://localhost:18000/admin/event_routing_backends/routerconfiguration/add/ + +#. Select ``Backend name`` (``xapi`` or ``caliper``). + +#. Add ``Route URL``; the HTTP endpoint where events are to be received. + +#. Select ``Auth Scheme`` (``Basic`` or ``Bearer`` or ``None``). For ``Basic`` authentication, add ``username`` and ``password``. For ``Bearer`` authentication, add ``Token``. + +#. Add ``Configurations`` comprising of following configuration items as json: + + #. ``override_args``: Accepts set of key:value pairs that will be added at the root level of the json of the event being routed. If the any of the keys already exist at the root level, their value will be overridden. Please note that for ``caliper`` backend, these changes will be made in the envelope. + + #. ``match_params``: This can be used to filter events based on values of keys in the original edX events. Regular expressions can be used for values. + + #. ``headers``: Additional headers can be specified here for ``caliper`` backend only. + +A sample configuration for routing Caliper events having content organisation as ``edX`` AND course run is 2021 AND event name starts with ``problem`` OR event name contains ``video``, with override arguments and additional headers: + +.. code-block:: JSON + + { + "override_args":{ + "sensor_id":"sensor@example.com" + }, + "headers":{ + "test":"header" + }, + "match_params":{ + "course_id":"^.*course-v.:edX\\+.*\\+2021.*$", + "name":[ + "^problem.*", + "video" + ] + } + } + +A sample configuration for routing xAPI events if the enterprise is ``org_XYZ`` AND event name is ``edx.course.grade.passed.first_time`` OR ``edx.course.enrollment.activated``: + +.. code-block:: JSON + + { + "match_params":{ + "enterprise_uuid":"org_XYZ", + "name":[ + "edx.course.grade.passed.first_time", + "edx.course.enrollment.activated" + ] + } + } + +Backends configuration +---------------------- + +By default, both ``caliper`` and ``xapi`` backends are configured with ``NameWhitelistProcessor`` that filters all the events currently supported. Users can override default backends to change filter type and name of the events to be filtered. + +A sample override for ``caliper`` backend is presented below. Here we are allowing only enrollment, ``seek_video`` and ``edx.video.position.changed`` events to be filtered through `RegexFilter`_ to ``caliper`` backend. + +.. _RegexFilter: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/regex_filter.py + + .. code-block:: python + + EVENT_TRACKING_BACKENDS.update({ + 'caliper': { + 'ENGINE': 'eventtracking.backends.async_routing.AsyncRoutingBackend', + 'OPTIONS': { + 'backend_name': 'caliper', + 'processors': [ + { + 'ENGINE': 'eventtracking.processors.regex_filter.RegexFilter', + 'OPTIONS': { + 'filter_type': 'allowlist', + 'regular_expressions': [ + 'edx.course.enrollment.*', + 'seek_video', + 'edx.video.position.changed' + ] + } + } + ], + 'backends': { + 'caliper': { + 'ENGINE': 'event_routing_backends.backends.events_router.EventsRouter', + 'OPTIONS': { + 'processors': [ + { + 'ENGINE': 'event_routing_backends.processors.caliper.transformer_processor.CaliperProcessor', + 'OPTIONS': {} + }, + { + 'ENGINE': 'event_routing_backends.processors.caliper.envelope_processor.CaliperEnvelopeProcessor', + 'OPTIONS': { + 'sensor_id': 'http://example.com/sensors' + } + } + ], + 'backend_name': 'caliper' + } + } + } + } + } + }) + +A sample override for ``xapi`` backend is presented below. Here we are allowing only enrollment, ``edx.course.grade.passed.first_time`` and ``edx.ui.lms.sequence.tab_selected`` events to be filtered through `NameWhitelist`_ to ``xapi`` backend. + +.. _NameWhiteList: https://github.com/openedx/event-tracking/blob/master/eventtracking/processors/whitelist.py + + .. code-block:: python + + EVENT_TRACKING_BACKENDS.update({ + 'xapi': { + 'ENGINE': 'eventtracking.backends.async_routing.AsyncRoutingBackend', + 'OPTIONS': { + 'backend_name': 'xapi', + 'processors': [ + { + 'ENGINE': 'eventtracking.processors.whitelist.NameWhitelistProcessor', + 'OPTIONS': { + 'whitelist': [ + 'edx.course.enrollment.activated', + 'edx.course.enrollment.deactivated', + 'edx.course.grade.passed.first_time', + 'edx.ui.lms.sequence.tab_selected', + ] + } + } + ], + 'backends': { + 'xapi': { + 'ENGINE': 'event_routing_backends.backends.events_router.EventsRouter', + 'OPTIONS': { + 'processors': [ + { + 'ENGINE': 'event_routing_backends.processors.xapi.transformer_processor.XApiProcessor', + 'OPTIONS': {} + } + ], + 'backend_name': 'xapi' + } + } + } + } + } + } + +Batching Configuration +---------------------- + +Batching of events can be configured using the following settings: + +#. ``EVENT_ROUTING_BACKEND_BATCHING_ENABLED``: If set to ``True``, events will be batched before being routed. Default is ``False``. +#. ``EVENT_ROUTING_BACKEND_BATCH_SIZE``: Maximum number of events to be batched together. Default is 100. +#. ``EVENT_ROUTING_BACKEND_BATCH_INTERVAL``: Time interval (in seconds) after which events will be ent, whether or not the batch size criteria is met. Default is 60 seconds. + +Batching is done in the ``EventsRouter`` backend. If ``EVENT_ROUTING_BACKEND_BATCHING_ENABLED`` is set to ``True``, then events will be batched together and routed to the configured routers after the specified interval or when the batch size is reached, whichever happens first. + +In case of downtimes or network issues, events will be queued again to avoid data loss. However, there is no guarantee that the events will be routed in the same order as they were received. + +Event bus configuration +----------------------- + +The event bus backend can be configured as the producer of the events. In that case, the events will be consumed from the event bus and routed to the configured routers via event bus consumers. The event bus backend can be configured in your edx-platform settings as follows: + +.. code-block:: python + + EVENT_TRACKING_BACKENDS["xapi"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" + EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"]["backends"]["xapi"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" + EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"].pop("backend_name") + if "openedx_events" not in INSTALLED_APPS: + INSTALLED_APPS.append("openedx_events") + SEND_TRACKING_EVENT_EMITTED_SIGNAL = True + EVENT_BUS_PRODUCER_CONFIG = { + "org.openedx.analytics.tracking.event.emitted.v1": { + "analytics": { + "event_key_field": "tracking_log.name", "enabled": True + } + } + } + +Once the event bus producer has been configured, the event bus consumer can be started using the following command: + +.. code-block:: bash + + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "event_routing_backends"}' diff --git a/docs/technical_documentation/quickstarts/index.rst b/docs/technical_documentation/quickstarts/index.rst new file mode 100644 index 00000000..c8ab1600 --- /dev/null +++ b/docs/technical_documentation/quickstarts/index.rst @@ -0,0 +1,9 @@ +Quickstarts +########### + +.. toctree:: + :maxdepth: 2 + + installation + configuration + testing diff --git a/docs/technical_documentation/quickstarts/installation.rst b/docs/technical_documentation/quickstarts/installation.rst new file mode 100644 index 00000000..0cefb6fa --- /dev/null +++ b/docs/technical_documentation/quickstarts/installation.rst @@ -0,0 +1,10 @@ +Installation +############ + +Install event routing backends library or add it to private requirements of your virtual environment ( ``requirements/private.txt`` ). + +#. Run ``pip install edx-event-routing-backends``. + +#. Run migrations ( ``python manage.py lms migrate`` ). + +#. Restart LMS service and celery workers of edx-platform. diff --git a/docs/testing.rst b/docs/technical_documentation/quickstarts/testing.rst similarity index 96% rename from docs/testing.rst rename to docs/technical_documentation/quickstarts/testing.rst index 05f8d895..8bff3ce1 100644 --- a/docs/testing.rst +++ b/docs/technical_documentation/quickstarts/testing.rst @@ -1,7 +1,5 @@ -.. _chapter-testing: - Testing -======= +####### event-routing-backends has an assortment of test cases and code quality checks to catch potential problems during development. To run them all in the