diff --git a/.gitignore b/.gitignore index d7c55fb..f063246 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,182 @@ files/ **.log + +# Don't commit these files right now. It can cause grief with folks that run different +# versions of python, etc. +Pipfile +Pipfile.lock + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 8fdc62e..0000000 --- a/Pipfile +++ /dev/null @@ -1,18 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[dev-packages] -pylint = "*" -jedi = "*" - -[packages] -requests = "*" -markdownify = "*" -"bs4" = "*" -canvasapi = "*" -pathvalidate = "*" - -[requires] -python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index e58976f..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,228 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "58bbea71191ae123c2d0a77e0f07ba5cc150f8b6518e342fdb9d0753b222332c" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "beautifulsoup4": { - "hashes": [ - "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf", - "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891" - ], - "markers": "python_version >= '3.1'", - "version": "==4.10.0" - }, - "bs4": { - "hashes": [ - "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" - ], - "index": "pypi", - "version": "==0.0.1" - }, - "canvasapi": { - "hashes": [ - "sha256:5087db773cac9d92f4f4609b3c160dbeeceb636801421808afee2d438bc43f62" - ], - "index": "pypi", - "version": "==2.2.0" - }, - "certifi": { - "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" - ], - "version": "==2021.5.30" - }, - "charset-normalizer": { - "hashes": [ - "sha256:7098e7e862f6370a2a8d1a6398cd359815c45d12626267652c3f13dec58e2367", - "sha256:fa471a601dfea0f492e4f4fca035cd82155e65dc45c9b83bf4322dfab63755dd" - ], - "markers": "python_version >= '3.0'", - "version": "==2.0.5" - }, - "idna": { - "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" - ], - "markers": "python_version >= '3.0'", - "version": "==3.2" - }, - "markdownify": { - "hashes": [ - "sha256:153c9e739544a5dc086dbc55709c5019532f81052fbdf0dc937f7835009110dd", - "sha256:a32d7d6b0801f81d12a44533093a5681b402c6b7e9a7049512d26697517eb795" - ], - "index": "pypi", - "version": "==0.9.4" - }, - "pathvalidate": { - "hashes": [ - "sha256:3c9bd94c7ec23e9cfb211ffbe356ae75f979d6c099a2c745ee9490f524f32468", - "sha256:f5dde7efeeb4262784c5e1331e02752d07c1ec3ee5ea42683fe211155652b808" - ], - "index": "pypi", - "version": "==2.4.1" - }, - "pytz": { - "hashes": [ - "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", - "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" - ], - "version": "==2021.1" - }, - "requests": { - "hashes": [ - "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", - "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" - ], - "index": "pypi", - "version": "==2.26.0" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, - "soupsieve": { - "hashes": [ - "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc", - "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b" - ], - "markers": "python_version >= '3.6'", - "version": "==2.2.1" - }, - "urllib3": { - "hashes": [ - "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", - "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.6" - } - }, - "develop": { - "astroid": { - "hashes": [ - "sha256:dcc06f6165f415220013801642bd6c9808a02967070919c4b746c6864c205471", - "sha256:fe81f80c0b35264acb5653302ffbd935d394f1775c5e4487df745bf9c2442708" - ], - "markers": "python_version ~= '3.6'", - "version": "==2.8.0" - }, - "isort": { - "hashes": [ - "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899", - "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2" - ], - "markers": "python_version < '4.0' and python_full_version >= '3.6.1'", - "version": "==5.9.3" - }, - "jedi": { - "hashes": [ - "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93", - "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707" - ], - "index": "pypi", - "version": "==0.18.0" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653", - "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61", - "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2", - "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837", - "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3", - "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43", - "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726", - "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3", - "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587", - "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8", - "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a", - "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd", - "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f", - "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad", - "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4", - "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b", - "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf", - "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981", - "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741", - "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e", - "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93", - "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.6.0" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "parso": { - "hashes": [ - "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398", - "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22" - ], - "markers": "python_version >= '3.6'", - "version": "==0.8.2" - }, - "platformdirs": { - "hashes": [ - "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f", - "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648" - ], - "markers": "python_version >= '3.6'", - "version": "==2.3.0" - }, - "pylint": { - "hashes": [ - "sha256:0f358e221c45cbd4dad2a1e4b883e75d28acdcccd29d40c76eb72b307269b126", - "sha256:2c9843fff1a88ca0ad98a256806c82c5a8f86086e7ccbdb93297d86c3f90c436" - ], - "index": "pypi", - "version": "==2.11.1" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.10.2" - }, - "typing-extensions": { - "hashes": [ - "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e", - "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7", - "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34" - ], - "markers": "python_version < '3.10'", - "version": "==3.10.0.2" - }, - "wrapt": { - "hashes": [ - "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" - ], - "version": "==1.12.1" - } - } -} diff --git a/README.md b/README.md index fae2c48..9403af1 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,43 @@ This repo hosts a simple python script to download all available files in all courses on your Canvas page. ## Dependencies - - Python 3.6+ - - [requests](https://pypi.org/project/requests/) - - [markdownify](https://github.com/matthewwithanm/python-markdownify) - - [bs4](https://pypi.org/project/bs4/) -If you use [pipenv](https://github.com/pypa/pipenv) you can just run `pipenv install` to setup the environment. +- Python 3.11+ +- [setuptools-scm](https://pypi.org/project/setuptools-scm/) >= 8.0 + +### pipenv + +#### Installation + +```shell +pipenv install . +``` + +#### Direct Usage + +```shell +pipenv run canvas-scraper ... +``` + +### pipx + +#### Installation + +```shell +pipx install . +``` ## Usage ```shell -python canvas-scraper.py +canvas-scraper ``` -For info on how to get an API key please refer to the [Canvas Dev course](https://canvas.instructure.com/courses/785215/pages/getting-started-with-the-api) +## Setup Directions + +For info on how to get an API key please refer to the [Canvas Dev course](https://canvas.instructure.com/courses/785215/pages/getting-started-with-the-api). + +## TODO -## Todo - - Add async option - - Add support for more item types - - Store returned json data from Canvas API + - Add async option. + - Add support for more item types. + - Store returned JSON data from Canvas API. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dfd8aa6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools>=65", "setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[project] +name = "canvas_file_scraper" +version = "0.0.1" +dependencies = [ + "beautifulsoup4~=4.14.3", + "canvasapi~=3.5.0", + "markdownify~=1.2.2", + "pathvalidate~=3.3.1", + "requests~=2.32.5" +] +requires-python = ">=3.11" +authors = [ + { name = "Enji Cooper", email = "yaneurabeya@gmail.com" } +] +maintainers = [ + { name = "Enji Cooper", email = "yaneurabeya@gmail.com" } +] +description = "Mirroring tool for UBC Canvas." +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["selenium", "wget"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15" +] + +[project.scripts] +canvas-file-scraper = "canvas_file_scraper.__main__:main" + +[project.urls] +Repository = "https://github.com/Gigahawk/canvas-file-scraper/" +Issues = "https://github.com/Gigahawk/canvas-file-scraper/issues" + +[tool.coverage.html] +directory = "htmlcov" + +[tool.coverage.run] +branch = true +source = ["src/"] + +[tool.pytest.ini_options] +timeout = 10 + +[tool.setuptools] +package-dir = {"" = "src/"} + +[tool.tox] +legacy_tox_ini = """ +[tox] +min_version = 4.0 +env_list = + coverage + py311 + py312 + py313 + py314 + py315 + +[testenv] +deps = + pytest + pytest-timeout +commands = + {envpython} -m pytest {posargs:-v tests} + +[testenv:coverage] +deps = + coverage + pytest + pytest-cov + pytest-timeout +commands = + {envpython} -m pytest --cov=canvas_file_scraper --cov-fail-under=80 --cov-report=html --cov-report=term \ + {posargs:-vvvv tests} + +[testenv:type] +deps = + mypy +commands = + {envpython} -m mypy {posargs:src} +""" diff --git a/canvas_file_scraper/__init__.py b/src/canvas_file_scraper/__init__.py similarity index 100% rename from canvas_file_scraper/__init__.py rename to src/canvas_file_scraper/__init__.py diff --git a/main.py b/src/canvas_file_scraper/__main__.py similarity index 50% rename from main.py rename to src/canvas_file_scraper/__main__.py index cfcc101..da33e3f 100644 --- a/main.py +++ b/src/canvas_file_scraper/__main__.py @@ -1,16 +1,10 @@ import argparse import logging import sys -import requests -import os -import pprint -import json -from bs4 import BeautifulSoup -import re + from canvas_file_scraper.scraper import CanvasScraper -log_formatter = logging.Formatter( - "[%(levelname)-5.5s][%(name)s] %(message)s") +log_formatter = logging.Formatter("[%(levelname)-5.5s][%(name)s] %(message)s") logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -25,25 +19,47 @@ def main(): parser = argparse.ArgumentParser( - description='Grabs all files for all courses on Canvas') + description="Grabs all files for all courses on Canvas", + ) parser.add_argument( - 'canvas_api_key', metavar='key', type=str, - help='Canvas API Key obtained from "settings"') + "canvas_api_key", + metavar="key", + type=str, + help='Canvas API Key obtained from "settings"', + ) parser.add_argument( - '-u', '--canvas_url', type=str, default='canvas.ubc.ca', - help='Canvas base URL (default: canvas.ubc.ca)') + "-u", + "--canvas_url", + type=str, + default="canvas.ubc.ca", + help="Canvas base URL (default: canvas.ubc.ca)", + ) parser.add_argument( - '-v', '--video', action='store_true', - help='Enable kaltura video downloads (Warning: spacetime intensive)') + "-v", + "--video", + action="store_true", + help="Enable kaltura video downloads (Warning: spacetime intensive)", + ) parser.add_argument( - '-o', '--overwrite', type=str, default='no', - help='Can be one of "yes", "no", or "ask", (default: no)') + "-o", + "--overwrite", + type=str, + default="no", + help='Can be one of "yes", "no", or "ask", (default: no)', + ) parser.add_argument( - '-d', '--directory', type=str, default='./files', - help='Directory to store downloaded files in (default: ./files)') + "-d", + "--directory", + type=str, + default="./files", + help="Directory to store downloaded files in (default: ./files)", + ) parser.add_argument( - '-m', '--markdown', action="store_true", - help='Convert downloaded pages to markdown') + "-m", + "--markdown", + action="store_true", + help="Convert downloaded pages to markdown", + ) args = parser.parse_args() scraper = CanvasScraper( @@ -53,7 +69,8 @@ def main(): args.overwrite, args.video, args.markdown, - logger) + logger, + ) logger.info("Starting scrape") scraper.scrape() diff --git a/canvas_file_scraper/scraper.py b/src/canvas_file_scraper/scraper.py similarity index 70% rename from canvas_file_scraper/scraper.py rename to src/canvas_file_scraper/scraper.py index 82ff31a..ef52a20 100644 --- a/canvas_file_scraper/scraper.py +++ b/src/canvas_file_scraper/scraper.py @@ -1,22 +1,25 @@ -import types -import re -import os -import requests -from requests.exceptions import MissingSchema -import logging +"""Core scraper logic.""" + +import collections import json -from tempfile import TemporaryFile -from pathvalidate import sanitize_filename +import logging +import os +import re +import types import urllib +from tempfile import TemporaryFile + +import requests from bs4 import BeautifulSoup -from markdownify import markdownify as md from canvasapi import Canvas -from canvasapi.exceptions import Unauthorized, ResourceDoesNotExist - from canvasapi.canvas_object import CanvasObject +from canvasapi.exceptions import Forbidden, ResourceDoesNotExist, Unauthorized from canvasapi.file import File from canvasapi.paginated_list import PaginatedList from canvasapi.util import combine_kwargs +from markdownify import markdownify as md +from pathvalidate import sanitize_filename +from requests.exceptions import MissingSchema class MediaObject(CanvasObject): @@ -28,20 +31,19 @@ def get_media_objects(self, *args, **kwargs): MediaObject, self._requester, "GET", - "courses/{}/media_objects".format(self.id), + f"courses/{self.id}/media_objects", {"course_id": self.id}, _kwargs=combine_kwargs(**kwargs), ) - class CanvasScraper: def __init__( - self, base_url, api_key, path, overwrite, - videos, markdown, logger=None): + self, base_url, api_key, path, overwrite, videos, markdown, logger=None, + ): self.api_key = api_key self.base_url = self._create_base_url(base_url) - self.headers = {'Authorization': f'Bearer {self.api_key}'} + self.headers = {"Authorization": f"Bearer {self.api_key}"} self._path = path self.overwrite = overwrite self.videos = videos @@ -49,7 +51,7 @@ def __init__( self._logger = logger self._canvas = Canvas(self.base_url, self.api_key) self.user = self._canvas.get_current_user() - self.visited_page_links = [] + self.visited_page_links = collections.Counter() if not self._logger: self._logger = logging @@ -65,8 +67,8 @@ def scrape(self): print(c) except AttributeError: print("Null course") - #import pdb - #pdb.set_trace() + # import pdb + # pdb.set_trace() self.recurse_course(c) def recurse_course(self, course): @@ -79,14 +81,17 @@ def recurse_course(self, course): try: external_tools = course.get_external_tools() external_tools = list(external_tools) - self.logger.info(str(course.name)) - self.logger.info(external_tools) + self.logger.info( + "course name=%r; external_tools=%r", + str(course.name), + external_tools, + ) if external_tools: import pdb + pdb.set_trace() - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"External tools not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("External tools not accessible", exc_info=True) self.push_raw(f"assignments_{course.id}", "assignments", 0) try: @@ -97,9 +102,8 @@ def recurse_course(self, course): self.handle_assignment(a) finally: self.pop() - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Assignments not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Assignments not accessible", exc_info=True) finally: self.pop() @@ -112,9 +116,8 @@ def recurse_course(self, course): self.handle_page(p) finally: self.pop() - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Pages not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Pages not accessible", exc_info=True) finally: self.pop() @@ -126,26 +129,22 @@ def recurse_course(self, course): if self._dl_page(fp, fp_path) and self.markdown: self._dl_page_data(fp_path, course._requester) self._markdownify(fp_path, fp_md_path) - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Front page not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Front page not accessible", exc_info=True) try: modules = course.get_modules() for m in modules: self.recurse_module(m) - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Modules not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Modules not accessible", exc_info=True) try: groups = course.get_groups() for g in groups: self.recurse_group(g) - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Groups not accesible") - + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Groups not accessible", exc_info=True) self.scrape_files(course) @@ -175,7 +174,7 @@ def scrape_files(self, obj): for f in folders: self.recurse_folder(f) except Unauthorized: - self.logger.warning(f"Files not accesible") + self.logger.warning("Files not accessible", exc_info=True) finally: self.pop() @@ -191,12 +190,14 @@ def scrape_media(self, obj): self.handle_media_video(m) else: self.logger.warning( - f"Media '{m.title}' type {m.media_type} is unsupported") + "Media %r type {m.media_type} is unsupported", + m.m_title, + ) import pdb + pdb.set_trace() - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(e) - self.logger.warning(f"Media objects not accesible") + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("Media objects not accessible", exc_info=True) finally: self.pop() @@ -211,23 +212,22 @@ def recurse_folder(self, folder): except AttributeError: try: f_name = f.display_name - except Exception as e: + except Exception: import pdb + pdb.set_trace() f_path = os.path.join(self.path, f_name) if self._should_write(f_path): - self.logger.info(f"Downloading {f_path}") + self.logger.info("Downloading %s", f_path) try: f.download(f_path) - self.logger.info(f"{f_path} downloaded") - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(f"file not accesible") - self.logger.warning(str(e)) - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.warning(f"folder not accesible") - self.logger.warning(str(e)) + self.logger.info("%s downloaded", f_path) + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("file not accessible", exc_info=True) + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.warning("folder not accessible", exc_info=True) finally: self.pop() @@ -257,17 +257,17 @@ def recurse_item(self, item): self.handle_quiz(item) elif item.type == "SubHeader": # TODO: Assuming you can't nest subheaders, it's probably enough - # to just pop the stack if the top contains a subheader, and then + # to just pop the stack if the top contains a subheader, and then # push a new folder for each subheader. - self.logger.warning( - "SubHeader's are not supported for now, skipping") - #self.handle_subheader(item) + self.logger.warning("SubHeader's are not supported for now, skipping") + # self.handle_subheader(item) elif item.type == "ExternalUrl": self.logger.info("Handling external URL") self.handle_external_url(item) else: - self.logger.warning(f"Unsupported type {item.type}") + self.logger.warning("Unsupported type %s", item.type) import pdb + pdb.set_trace() finally: self.pop() @@ -278,23 +278,22 @@ def handle_external_url(self, item): if self._should_write(file_path): with open(file_path, "w") as f: f.write(url) - self.logger.info(f"{file_path} downloaded") + self.logger.info("%s downloaded", file_path) def handle_file(self, item): file_name = item.title file_url = item.url file_path = os.path.join(self.path, file_name) requester = item._requester - self.logger.info(f"Downloading {file_name}") - self._dl_canvas_file( - file_url, file_path, requester) + self.logger.info("Downloading %s", file_name) + self._dl_canvas_file(file_url, file_path, requester) def handle_media_video(self, item): media_name = item.title media_path = os.path.join(self.path, media_name) sources = item.media_sources - sources.sort(key=lambda s: int(s['size']), reverse=True) - media_url = sources[0]['url'] + sources.sort(key=lambda s: int(s["size"]), reverse=True) + media_url = sources[0]["url"] self._dl(media_url, media_path) def handle_page(self, item): @@ -304,16 +303,17 @@ def handle_page(self, item): url = item.url else: self.logger.error("Could not get url for page item") - import pdb;pdb.set_trace() - page = self._canvas.get_course( - item.course_id).get_page(url) + import pdb + + pdb.set_trace() + page = self._canvas.get_course(item.course_id).get_page(url) try: page_body = page.body except AttributeError: if page.locked_for_user: - self.logger.info("Page locked, reason:") - self.logger.info(page.lock_explanation) - self.logger.error("Page not accessible") + self.logger.info("Page locked; reason: %s", page.lock_explanation) + else: + self.logger.exception("Page not accessible") return page_path = os.path.join(self.path, "page.html") @@ -330,13 +330,14 @@ def handle_assignment(self, item): asn_id = item.id else: self.logger.error("Could not get url for assignment item") - import pdb;pdb.set_trace() + import pdb + + pdb.set_trace() page_path = os.path.join(self.path, "assignment.html") page_md_path = os.path.join(self.path, "assignment.md") json_path = os.path.join(self.path, "assignment.json") - assignment = self._canvas.get_course( - item.course_id).get_assignment(asn_id) + assignment = self._canvas.get_course(item.course_id).get_assignment(asn_id) self._dl_obj(assignment, json_path) @@ -353,8 +354,7 @@ def handle_quiz(self, item): page_path = os.path.join(self.path, "quiz.html") page_md_path = os.path.join(self.path, "quiz.md") json_path = os.path.join(self.path, "quiz.json") - quiz = self._canvas.get_course( - item.course_id).get_quiz(item.content_id) + quiz = self._canvas.get_course(item.course_id).get_quiz(item.content_id) page = quiz.description if page: if self.markdown and self._dl_page(page, page_path): @@ -393,7 +393,7 @@ def push_raw(self, name, type, id): self._push_logger(f"{type}_{id}") self._push_name(name) self._push_id(id) - self.logger.info(name) + self.logger.debug("push_raw: %s", name) def pop(self): self._pop_logger() @@ -401,7 +401,7 @@ def pop(self): self._pop_id() def get_all_objects(self, url): - self.logger.debug(f"Grabbing all pages for {url}") + self.logger.debug("Grabbing all pages for %s", url) objects = [] page = 1 while True: @@ -409,7 +409,7 @@ def get_all_objects(self, url): if not r.json(): break objects.extend(r.json()) - self.logger.debug(f"Grabbed page {page}") + self.logger.debug("Grabbed page %s", page) page += 1 return objects @@ -419,8 +419,7 @@ def logger(self): @property def path(self): - return os.path.join( - self._path, *[sanitize_filename(n) for n in self._names]) + return os.path.join(self._path, *[sanitize_filename(n) for n in self._names]) @property def name(self): @@ -449,14 +448,15 @@ def _modules_url(self, course_id): return f"{self._course_url(course_id)}/modules" def _kaltura_manifest_url(self, base_url, entry_id, flavor_id): - base_url = base_url[:base_url.index("embedIframeJs")] + base_url = base_url[: base_url.index("embedIframeJs")] return os.path.join( base_url, "playManifest/entryId", str(entry_id), "flavorIds", str(flavor_id), - "format/applehttp/protocol/https/a.m3u8") + "format/applehttp/protocol/https/a.m3u8", + ) def _get(self, url, params=None): return requests.get(url, params=params, headers=self.headers) @@ -465,43 +465,47 @@ def _mkd(self, path): return os.makedirs(path, exist_ok=True) def _dl(self, url, path): - if self._should_write(path): - try: - self.logger.info(f"Downloading {path}") - r = self._get(url) - with open(path, "wb") as f: - f.write(r.content) - self.logger.info(f"{path} downloaded") - return True - except MissingSchema as e: - self.logger.error(f"{url} is not a valid url") - return False - except Exception as e: - self.logger.error("file download failed") - import pdb - pdb.set_trace() - self.logger.error(e) + if not self._should_write(path): + return + try: + self.logger.info("Downloading %s", path) + r = self._get(url) + with open(path, "wb") as f: + f.write(r.content) + except MissingSchema: + self.logger.error("%s is not a valid url", url) + return False + except Exception: + self.logger.exception("file download failed") + import pdb + + pdb.set_trace() + else: + self.logger.info("%s downloaded", path) + return True def _dl_page(self, page, path): - if self._should_write(path): - with open(path, "w") as f: - f.writelines(page) - self.logger.info(f"{path} downloaded") - return True + if not self._should_write(path): + return + with open(path, "w") as f: + f.writelines(page) + self.logger.info("%s downloaded", path) + return True def _dl_obj(self, obj, path): - if self._should_write(path): - with open(path, "w") as f: - json.dump(obj.__dict__, f, indent=2, default=str) - self.logger.info(f"{path} downloaded") + if not self._should_write(path): + return + with open(path, "w") as f: + json.dump(obj.__dict__, f, indent=2, default=str) + self.logger.info("%s downloaded", path) def _dl_page_data(self, src_path, requester): self.logger.info(f"Downloading page data for {src_path}") - with open(src_path, "r") as f: + with open(src_path) as f: src = f.read() soup = BeautifulSoup(src, "html.parser") - links = soup.find_all('a') + links = soup.find_all("a") if links: self._mkd(os.path.join(self.path, "files")) @@ -511,24 +515,28 @@ def _dl_page_data(self, src_path, requester): if not title: title = link.text if not href: - self.logger.warning(f"Link not found for title {title}") + self.logger.warning("Link not found for title %s", title) continue - self.logger.info(f"Downloading link for: {title}") + self.logger.info("Downloading link for: %s", link) self.logger.info(href) - if href in self.visited_page_links: - self.logger.warning("Page has been visited before, skipping") + if self.visited_page_links[href]: + self.logger.warning("Page has been visited before; skipping") continue - self.visited_page_links.append(href) - if link.get("class") and "instructure_file_link" in link["class"] and "canvas" in href: + self.visited_page_links[href] += 1 + if ( + link.get("class") + and "instructure_file_link" in link["class"] + and "canvas" in href + ): # This is necessary because files don't always show up # under the files section of a course for some reason - self.logger.info( - "Canvas file detected, using Canvas API for download") + self.logger.info("Canvas file detected; using Canvas API for download") try: self._dl_canvas_file( - href, os.path.join(self.path, "files"), requester) - except (Unauthorized, ResourceDoesNotExist) as e: - self.logger.error("Could not download file") + href, os.path.join(self.path, "files"), requester, + ) + except (Forbidden, Unauthorized, ResourceDoesNotExist): + self.logger.exception("Could not download file") elif href.startswith("mailto"): self.logger.info("mailto link detected, saving email") mail_path = os.path.join(self.path, "files", title) @@ -540,29 +548,30 @@ def _dl_page_data(self, src_path, requester): self.push_raw(f"page_{page_item.page_url}", "page", 0) try: self.handle_page(page_item) - except: - self.logger.info("Could not handle page item") + except Exception: + self.logger.exception("Could not handle page item") finally: self.pop() elif self._is_assignment_url(href): - self.logger.info("Canvas assignment detected, handling assignment") - assignment_item = self._assignment_url_to_item(href, requester) - self.push_raw(f"assignment_{assignment_item.content_id}", "assignment", 0) + self.logger.info("Canvas assignment detected; handling assignment") + assignment_item = self._assignment_url_to_item(href, requester) + self.push_raw( + f"assignment_{assignment_item.content_id}", "assignment", 0, + ) try: self.handle_assignment(assignment_item) - except: - self.logger.info("Could not handle assignment item") + except Exception: + self.logger.exception("Could not handle assignment item") finally: self.pop() else: - self.logger.warning( - "Non Canvas file link, attempting generic download") + self.logger.warning("Non Canvas file link, attempting generic download") dl_path = os.path.join(self.path, "files", title) self._dl(link["href"], dl_path) if self.videos: # Download Kaltura videos - videos = soup.find_all('iframe', **{'id': 'kaltura_player'}) + videos = soup.find_all("iframe", id="kaltura_player") for idx, video in enumerate(videos): video_path = os.path.join(self.path, "videos", f"{idx}.mp4") self._dl_video(video["src"], video_path) @@ -576,7 +585,7 @@ def _dl_canvas_file(self, url, path, requester): if not self._should_write(dl_path): return file.download(dl_path) - self.logger.info(f"{dl_path} downloaded") + self.logger.info("%s downloaded", dl_path) return True def _dl_video(self, base_url, path): @@ -584,57 +593,52 @@ def _dl_video(self, base_url, path): return # Get data from Kaltura iframe lines = requests.get(base_url).text.splitlines() - iframe_data = next( - (l for l in lines if "kalturaIframePackageData" in l), None) + iframe_data = next((l for l in lines if "kalturaIframePackageData" in l), None) if not iframe_data: - self.logger.warning(f"iframe data not found for {base_url}") + self.logger.warning("iframe data not found for %s", base_url) return # Ignore js syntax, pull json text out of line - iframe_data = iframe_data[iframe_data.index("{"):-1] + iframe_data = iframe_data[iframe_data.index("{") : -1] iframe_data = json.loads(iframe_data) try: - flavor_assets = (iframe_data["entryResult"] - ["contextData"] - ["flavorAssets"]) + flavor_assets = iframe_data["entryResult"]["contextData"]["flavorAssets"] except KeyError: - self.logger.warning(f"flavorAssets not found in {base_url}") + self.logger.warning("flavorAssets not found in %s", base_url, exc_info=True) return flavor_asset = next( - (f for f in flavor_assets if f.get("flavorParamsId") == 5), - None) + (f for f in flavor_assets if f.get("flavorParamsId") == 5), None, + ) if not flavor_asset: - self.logger.warning( - f"Could not find correct flavorAsset for {base_url}") + self.logger.warning("Could not find correct flavorAsset for %s", base_url) return try: entry_id = flavor_asset["entryId"] flavor_id = flavor_asset["id"] except KeyError: self.logger.warning( - f"Could not find keys inside flavorAsset for {base_url}") + "Could not find keys inside flavorAsset for %s", base_url, + ) return - manifest_url = self._kaltura_manifest_url( - base_url, entry_id, flavor_id) + manifest_url = self._kaltura_manifest_url(base_url, entry_id, flavor_id) lines = requests.get(manifest_url).text.splitlines() index_url = next((l for l in lines if "index" in l), None) if not index_url: - self.logger.warning( - f"Could not find index urlfor {base_url}") + self.logger.warning("Could not find index url for %s", base_url) return index = filter( - lambda l: not l.startswith("#"), - requests.get(index_url).text.splitlines()) + lambda l: not l.startswith("#"), requests.get(index_url).text.splitlines(), + ) streaming_url = index_url.replace("index.m3u8", "") with TemporaryFile() as tf: for i in index: - self.logger.info(f"Downloading video segment {i}") + self.logger.info("Downloading video segment %s", i) segment_url = os.path.join(streaming_url, i) tf.write(requests.get(segment_url).content) with open(path, "wb") as f: tf.seek(0) f.write(tf.read()) - self.logger.info(f"Downloaded {path} successfully") + self.logger.info("Downloaded %s successfully", path) def _is_page_url(self, url): page_regex = re.compile(r".+courses/\d+/pages/.+") @@ -663,21 +667,22 @@ def _url_to_item(self, url, requester, attrname): setattr(item, attrname, name) return item - def _markdownify(self, src_path, dest_path): if self._should_write(dest_path): - self.logger.info(f"Converting {src_path} to markdown") - with open(src_path, "r") as f: + self.logger.info("Converting %s to markdown", src_path) + with open(src_path) as f: src = f.read() with open(dest_path, "w") as f: f.writelines(md(src)) def _should_write(self, path): - if os.path.isfile(path) and self.overwrite is "no": - self.logger.debug(f"Skipping file {path}") + if os.path.isfile(path) and self.overwrite == "no": + self.logger.debug("Skipping file %s", path) return False - elif (self.overwrite is "ask" and - input(f"{path} already exists, overwrite? (y/n)") != "y"): + if ( + self.overwrite == "ask" + and input(f"{path} already exists, overwrite? (y/n)") != "y" + ): return False # Ensure folder exists before writing os.makedirs(os.path.dirname(path), exist_ok=True) @@ -701,7 +706,3 @@ def _push_id(self, id): def _pop_id(self): self._ids.pop(-1) - - - -