diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..cd1694374 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: MirrorX Release + +on: + push: + tags: + - "v*.*.*" + +jobs: + BuildWheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8.12 + - name: Get the version + id: get_version + run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} + - name: Install deps fro Pypi Release + run: | + pip -q install setuptools wheel twine + pip -q install -r requirements.txt + - name: Publish to pypi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* || exit 0 + - name: Upload all the data files to github Release + uses: softprops/action-gh-release@v1 + with: + name: MirrorX Release ${{ steps.get_version.outputs.VERSION }} + files: | + dist/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Sleep + run: sleep 60 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: token + password: ${{ secrets.GH_TOKEN }} + - name: Docker meta + id: metaraw + uses: docker/metadata-action@v3 + with: + images: ghcr.io/iamliquidx/mirrorx + tags: | + type=semver,pattern={{version}} + - name: Build and push Docker images + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64, linux/arm64/v8 + push: true + tags: ${{ steps.metaraw.outputs.tags }} + labels: ${{ steps.metaraw.outputs.labels }} diff --git a/.gitignore b/.gitignore index 86facd3c1..ac258eaab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,20 @@ config.env *auth_token.txt -*.pyc downloads/* download/* -data* .vscode .idea *.json *.pickle authorized_chats.txt log.txt -accounts/* \ No newline at end of file +accounts/* +*.pyc +dht6.dat +aria.conf +.gitignore +dht.dat +build +dist +*.spec +/venv/ diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ea40a2d28..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "vendor/cmrudl.py"] - path = vendor/cmrudl.py - url = https://github.com/JrMasterModelBuilder/cmrudl.py.git diff --git a/Dockerfile b/Dockerfile index 9d3313774..bd340f79c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,30 @@ -FROM ubuntu:18.04 +FROM python:3.9-slim -WORKDIR /usr/src/app -RUN chmod 777 /usr/src/app +WORKDIR / +# Deps -RUN apt-get -qq update -RUN apt-get -qq install -y python3 python3-pip rar unzip git aria2 g++ gcc autoconf automake \ - m4 libtool qt4-qmake make libqt4-dev libcurl4-openssl-dev \ - libcrypto++-dev libsqlite3-dev libc-ares-dev \ - libsodium-dev libnautilus-extension-dev \ - libssl-dev libfreeimage-dev swig curl pv jq ffmpeg locales python3-lxml +SHELL [ "/usr/bin/bash" , "-cel" ] -# Installing mega sdk python binding -ENV MEGA_SDK_VERSION '3.6.4' -RUN git clone https://github.com/meganz/sdk.git sdk -WORKDIR sdk -RUN git checkout v$MEGA_SDK_VERSION && ./autogen.sh && \ - ./configure --disable-silent-rules --enable-python --disable-examples && \ - make -j$(nproc --all) && cd bindings/python/ && \ - python3 setup.py bdist_wheel && cd dist/ && \ - pip3 install --no-cache-dir megasdk-$MEGA_SDK_VERSION-*.whl +RUN \ +[[ ${valid_arch:-aarch64 amd64 x86_64} =~ ${HOST_CPU_ARCH:=$(uname -m)} ]] \ + || echo 'unsupported cpu arch' && exit 1 +RUN \ +export HOST_CPU_ARCH=$(uname -m) \ +sed -i 's/main/main non-free/g' /etc/apt/sources.list && \ +apt-get -qq update && \ +apt-get -qq install -y tzdata curl aria2 p7zip-full p7zip-rar wget xz-utils libmagic-dev gcc libffi-dev nscd && \ +apt-get -y autoremove && rm -rf /var/lib/apt/lists/* && apt-get clean && \ +wget -q https://github.com/yzop/gg/raw/main/ffmpeg-git-${HOST_CPU_ARCH}-static.tar.xz && \ +tar -xf ff*.tar.xz && rm -rf *.tar.xz && \ +mv ff*/ff* /usr/local/bin/ && rm -rf ff* && \ +wget -q https://github.com/viswanathbalusu/megasdkrest/releases/latest/download/megasdkrest-${HOST_CPU_ARCH} -O /usr/local/bin/megasdkrest && \ +chmod a+x /usr/local/bin/megasdkrest && mkdir /app/ && chmod 777 /app/ && \ +pip3 install --no-cache-dir MirrorX && \ +apt-get purge -yqq gcc && apt-get -y autoremove && rm -rf /var/lib/apt/lists/* && apt-get clean -COPY requirements.txt . -COPY extract /usr/local/bin -RUN chmod +x /usr/local/bin/extract -RUN pip3 install --no-cache-dir -r requirements.txt -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -COPY . . -COPY netrc /root/.netrc -RUN chmod +x aria.sh - -CMD ["bash","start.sh"] +WORKDIR /app +CMD ["MirrorX"] +### diff --git a/README.md b/README.md index a9e837d7c..7d7bc2555 100644 --- a/README.md +++ b/README.md @@ -1,151 +1,38 @@ -# What is this repo about? -This is a telegram bot writen in python for mirroring files on the internet to our beloved Google Drive. - -# Inspiration -This project is heavily inspired from @out386 's telegram bot which is written in JS. - -# Features supported: -- Mirroring direct download links to google drive -- Mirroring Mega.nz links to google drive (In development stage) -- Mirror Telegram files to google drive -- Mirror all youtube-dl supported links -- Extract zip, rar, tar and many supported file types and uploads to google drive -- Copy files from someone's drive to your drive (using Rclone) -- Service account support in cloning and uploading -- Download progress -- Upload progress -- Download/upload speeds and ETAs -- Docker support -- Uploading To Team Drives. -- Index Link support - -# Bot commands to be set in botfather - -``` -mirror - Start Mirroring -tarmirror - Upload tar (zipped) file -unzipmirror - Extract files -clone - copy folder to drive -watch - mirror YT-DL support link -tarwatch - mirror youtube playlist link as tar -cancel - Cancel a task -cancelall - Cancel all tasks -list - [query] searches files in G-Drive -status - Get Mirror Status message -stats - Bot Usage Stats -help - Get Detailed Help -log - Bot Log [owner only] -``` - -# How to deploy? -Deploying is pretty much straight forward and is divided into several steps as follows: -## Installing requirements - -- Clone this repo: -``` -git clone https://github.com/magneto261290/magneto-python-aria mirror-bot/ -cd mirror-bot -``` - -- Install requirements -For Debian based distros -``` -sudo apt install python3 -sudo snap install docker -``` -- For Arch and it's derivatives: -``` -sudo pacman -S docker python -``` - -## Setting up config file -``` -cp config_sample.env config.env -``` -- Remove the first line saying: -``` -_____REMOVE_THIS_LINE_____=True -``` -Fill up rest of the fields. Meaning of each fields are discussed below: -- **BOT_TOKEN** : The telegram bot token that you get from @BotFather -- **GDRIVE_FOLDER_ID** : This is the folder ID of the Google Drive Folder to which you want to upload all the mirrors. -- **DOWNLOAD_DIR** : The path to the local folder where the downloads should be downloaded to -- **DOWNLOAD_STATUS_UPDATE_INTERVAL** : A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least) -- **OWNER_ID** : The Telegram user ID (not username) of the owner of the bot -- **AUTO_DELETE_MESSAGE_DURATION** : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages -- **IS_TEAM_DRIVE** : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. -- **USE_SERVICE_ACCOUNTS**: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. -- **INDEX_URL** : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' -- **API_KEY** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. -- **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org -- **USER_SESSION_STRING** : Session string generated by running: -- **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) -``` -python3 generate_string_session.py -``` -Note: You can limit maximum concurrent downloads by changing the value of MAX_CONCURRENT_DOWNLOADS in aria.sh. By default, it's set to 2 - -## Getting Google OAuth API credential file - -- Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) -- Go to the OAuth Consent tab, fill it, and save. -- Go to the Credentials tab and click Create Credentials -> OAuth Client ID -- Choose Other and Create. -- Use the download button to download your credentials. -- Move that file to the root of mirror-bot, and rename it to credentials.json -- Visit [Google API page](https://console.developers.google.com/apis/library) -- Search for Drive and enable it if it is disabled -- Finally, run the script to generate token file (token.pickle) for Google Drive: -``` -pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib -python3 generate_drive_token.py -``` -## Deploying - -- Start docker daemon (skip if already running): -``` -sudo dockerd -``` -- Build Docker image: -``` -sudo docker build . -t mirror-bot -``` -- Run the image: -``` -sudo docker run mirror-bot -``` - -# Using service accounts for uploading to avoid user rate limit -For Service Account to work, you must set USE_SERVICE_ACCOUNTS="True" in config file or environment variables -Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts -## Generating service accounts -Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) ---------------------------------- -Let us create only the service accounts that we need. -**Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. - -``` -Note: 1 service account can copy around 750gb a day, 1 project makes 100 service accounts so thats 75tb a day, for most users this should easily suffice. -``` - -`python3 gen_sa_accounts.py --quick-setup 1 --new-only` - -A folder named accounts will be created which will contain keys for the service accounts created - -NOTE: If you have created SAs in past from this script, you can also just re download the keys by running: -``` -python3 gen_sa_accounts.py --download-keys project_id -``` - -### Add all the service accounts to the Team Drive or folder -- Run: -``` -python3 add_to_team_drive.py -d SharedTeamDriveSrcID -``` - -# Youtube-dl authentication using .netrc file -For using your premium accounts in youtube-dl, edit the netrc file (in the root directory of this repository) according to following format: -``` -machine host login username password my_youtube_password -``` -where host is the name of extractor (eg. youtube, twitch). Multiple accounts of different hosts can be added each separated by a new line \ No newline at end of file +

+ Material Bread logo +

+ + +

+ +# This Is A Telegram Bot Written In Python For Mirroring Files On The Internet To Our Beloved Google Drive. +

+ +Here Are Some Things To Get You Started.👇 + + +## 👉[All The Feature Of This Bot Or What This Bot Can Do For You.](https://github.com/iamLiquidX/MirrorX/wiki/Feature-Or-What-This-Bot-Can-Do) + +## 👉[How To Deploy](https://github.com/iamLiquidX/MirrorX/wiki/How-To-Deploy) + +## 👉[Commands To Use The Bot](https://github.com/iamLiquidX/MirrorX/wiki/Commands-To-Use-This-Bot) + +## 👉[Modification Guide](https://github.com/iamLiquidX/MirrorX/wiki/Modification) + + +For The Most Recent Changes, Please Check The Changelog.👇 + +## 👉[Changelog](https://github.com/iamLiquidX/MirrorX/wiki/Changelog) + + + +# Credits 👇 + +1. [Shivam Jha aka lzzy12](https://github.com/lzzy12) & [JaskaranSM aka Zero Cool](https://github.com/jaskaranSM) - They Built This Bot From Scratch. +2. [Sreeraj V R](https://github.com/SVR666)- Added Inline Button, Added Support For Deleting File/Folders From GDrive, Search Results On Telegra.ph. +3. [Archie](https://github.com/archie9211) - Added Support For Extraction Of Archives, Fixed SSL Handshake Error, Update Trackers Dynamically. +4. [Magneto](https://github.com/magneto261290) - Added Alot Of Customization, Support For Custom File Names, Support For Password Protected Archives, Quality Selection Option In YTDL And Much More. +5. [KenHV](https://github.com/KenHV) - Many Fixes And Imporovements. +6. [Anos](https://github.com/destiny6520) - Modification/Customization Guide. +7. [Viswanath](https://github.com/nenokkadine) - Fixes & Improvements, Dockerfile Clean Up, DHT Support In Aria. +8. [breakdowns](https://github.com/breakdowns) - Source Code For Count,Zip/Unzip GDrive Links & Fembed. diff --git a/aria.bat b/aria.bat deleted file mode 100644 index 388080057..000000000 --- a/aria.bat +++ /dev/null @@ -1 +0,0 @@ -aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 --max-connection-per-server=10 --rpc-max-request-size=1024M --seed-time=0.01 --min-split-size=10M --follow-torrent=mem --split=10 --daemon=true --allow-overwrite=true diff --git a/aria.sh b/aria.sh deleted file mode 100755 index efd9ab0d7..000000000 --- a/aria.sh +++ /dev/null @@ -1,7 +0,0 @@ -export MAX_DOWNLOAD_SPEED=0 -export MAX_CONCURRENT_DOWNLOADS=3 -aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 \ - --max-connection-per-server=10 --rpc-max-request-size=1024M \ - --seed-time=0.01 --min-split-size=10M --follow-torrent=mem --split=10 \ - --daemon=true --allow-overwrite=true --max-overall-download-limit=$MAX_DOWNLOAD_SPEED \ - --max-overall-upload-limit=1K --max-concurrent-downloads=$MAX_CONCURRENT_DOWNLOADS diff --git a/bin/MirrorX b/bin/MirrorX new file mode 100644 index 000000000..7c40066fb --- /dev/null +++ b/bin/MirrorX @@ -0,0 +1,3 @@ +#!/bin/bash +pip3 install -U MirrorX +MirrorXBot \ No newline at end of file diff --git a/bin/extract b/bin/extract new file mode 100755 index 000000000..bd05693c6 --- /dev/null +++ b/bin/extract @@ -0,0 +1,194 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo "Usage: $(basename $0) FILES" + exit 1 +fi + +extract() { + arg="$1" + cd "$(dirname "$arg")" || exit + case "$arg" in + *.tar.bz2) + tar xjf "$arg" --one-top-level + local code=$? + ;; + *.tar.gz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.bz2) + bunzip2 "$arg" + local code=$? + ;; + *.gz) + gunzip "$arg" + local code=$? + ;; + *.tar) + tar xf "$arg" --one-top-level + local code=$? + ;; + *.tbz2) + (tar xjf "$arg" --one-top-level) + local code=$? + ;; + *.tgz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.zip) + a_dir=$(expr "$arg" : '\(.*\).zip') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.7z) + a_dir=$(expr "$arg" : '\(.*\).7z') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.Z) + uncompress "$arg" + local code=$? + ;; + *.rar) + a_dir=$(expr "$arg" : '\(.*\).rar') + mkdir "$a_dir" + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.iso) + a_dir=$(expr "$arg" : '\(.*\).iso') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.wim) + a_dir=$(expr "$arg" : '\(.*\).wim') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.cab) + a_dir=$(expr "$arg" : '\(.*\).cab') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.apm) + a_dir=$(expr "$arg" : '\(.*\).apm') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.arj) + a_dir=$(expr "$arg" : '\(.*\).arj') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.chm) + a_dir=$(expr "$arg" : '\(.*\).chm') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.cpio) + a_dir=$(expr "$arg" : '\(.*\).cpio') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.cramfs) + a_dir=$(expr "$arg" : '\(.*\).cramfs') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.deb) + a_dir=$(expr "$arg" : '\(.*\).deb') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.dmg) + a_dir=$(expr "$arg" : '\(.*\).dmg') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.fat) + a_dir=$(expr "$arg" : '\(.*\).fat') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.hfs) + a_dir=$(expr "$arg" : '\(.*\).hfs') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.lzh) + a_dir=$(expr "$arg" : '\(.*\).lzh') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.lzma) + a_dir=$(expr "$arg" : '\(.*\).lzma') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.lzma2) + a_dir=$(expr "$arg" : '\(.*\).lzma2') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.mbr) + a_dir=$(expr "$arg" : '\(.*\).mbr') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.msi) + a_dir=$(expr "$arg" : '\(.*\).msi') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.mslz) + a_dir=$(expr "$arg" : '\(.*\).mslz') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.nsis) + a_dir=$(expr "$arg" : '\(.*\).nsis') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.ntfs) + a_dir=$(expr "$arg" : '\(.*\).ntfs') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.rpm) + a_dir=$(expr "$arg" : '\(.*\).rpm') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.squashfs) + a_dir=$(expr "$arg" : '\(.*\).squashfs') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.udf) + a_dir=$(expr "$arg" : '\(.*\).udf') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.vhd) + a_dir=$(expr "$arg" : '\(.*\).vhd') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *.xar) + a_dir=$(expr "$arg" : '\(.*\).xar') + 7z x "$arg" -o"$a_dir" + local code=$? + ;; + *) + echo "'$arg' cannot be extracted via extract()" 1>&2 + exit 1 + ;; + esac + cd - || exit $? + exit $code +} + +extract "$1" diff --git a/bin/pextract b/bin/pextract new file mode 100644 index 000000000..297a0eafc --- /dev/null +++ b/bin/pextract @@ -0,0 +1,195 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo "Usage: $(basename $0) FILES" + exit 1 +fi + +extract() { + arg="$1" + pswd="$2" + cd "$(dirname "$arg")" || exit + case "$arg" in + *.tar.bz2) + tar xjf "$arg" --one-top-level + local code=$? + ;; + *.tar.gz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.bz2) + bunzip2 "$arg" + local code=$? + ;; + *.gz) + gunzip "$arg" + local code=$? + ;; + *.tar) + tar xf "$arg" --one-top-level + local code=$? + ;; + *.tbz2) + (tar xjf "$arg" --one-top-level) + local code=$? + ;; + *.tgz) + tar xzf "$arg" --one-top-level + local code=$? + ;; + *.zip) + a_dir=$(expr "$arg" : '\(.*\).zip') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.7z) + a_dir=$(expr "$arg" : '\(.*\).7z') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.Z) + uncompress "$arg" + local code=$? + ;; + *.rar) + a_dir=$(expr "$arg" : '\(.*\).rar') + mkdir "$a_dir" + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.iso) + a_dir=$(expr "$arg" : '\(.*\).iso') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.wim) + a_dir=$(expr "$arg" : '\(.*\).wim') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.cab) + a_dir=$(expr "$arg" : '\(.*\).cab') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.apm) + a_dir=$(expr "$arg" : '\(.*\).apm') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.arj) + a_dir=$(expr "$arg" : '\(.*\).arj') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.chm) + a_dir=$(expr "$arg" : '\(.*\).chm') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.cpio) + a_dir=$(expr "$arg" : '\(.*\).cpio') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.cramfs) + a_dir=$(expr "$arg" : '\(.*\).cramfs') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.deb) + a_dir=$(expr "$arg" : '\(.*\).deb') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.dmg) + a_dir=$(expr "$arg" : '\(.*\).dmg') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.fat) + a_dir=$(expr "$arg" : '\(.*\).fat') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.hfs) + a_dir=$(expr "$arg" : '\(.*\).hfs') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.lzh) + a_dir=$(expr "$arg" : '\(.*\).lzh') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.lzma) + a_dir=$(expr "$arg" : '\(.*\).lzma') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.lzma2) + a_dir=$(expr "$arg" : '\(.*\).lzma2') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.mbr) + a_dir=$(expr "$arg" : '\(.*\).mbr') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.msi) + a_dir=$(expr "$arg" : '\(.*\).msi') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.mslz) + a_dir=$(expr "$arg" : '\(.*\).mslz') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.nsis) + a_dir=$(expr "$arg" : '\(.*\).nsis') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.ntfs) + a_dir=$(expr "$arg" : '\(.*\).ntfs') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.rpm) + a_dir=$(expr "$arg" : '\(.*\).rpm') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.squashfs) + a_dir=$(expr "$arg" : '\(.*\).squashfs') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.udf) + a_dir=$(expr "$arg" : '\(.*\).udf') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.vhd) + a_dir=$(expr "$arg" : '\(.*\).vhd') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *.xar) + a_dir=$(expr "$arg" : '\(.*\).xar') + 7z x "$arg" -o"$a_dir" -p"$pswd" + local code=$? + ;; + *) + echo "'$arg' cannot be extracted via extract()" 1>&2 + exit 1 + ;; + esac + cd - || exit $? + exit $code +} + +extract "$1" "$2" \ No newline at end of file diff --git a/bot/__init__.py b/bot/__init__.py index c08dee660..756cf6675 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -2,11 +2,24 @@ import os import threading import time +import random +import string +import subprocess +import pkgutil +import pathlib +import sys import aria2p +import requests import telegram.ext as tg from dotenv import load_dotenv +from pyrogram import Client +from telegraph import Telegraph +from megasdkrestclient import MegaSdkRestClient, errors as mega_err + import socket +import faulthandler +faulthandler.enable() socket.setdefaulttimeout(600) @@ -36,6 +49,35 @@ def getConfig(name: str): exit() except KeyError: pass +CWD = os.getcwd() +ariaconfig = pkgutil.get_data("bot", "data/aria.conf").decode() +dhtfile = pkgutil.get_data("bot", "data/dht.dat") +dht6file = pkgutil.get_data("bot", "data/dht6.dat") +with open("dht.dat", "wb+") as dht: + dht.write(dhtfile) +with open("dht6.dat", "wb+") as dht6: + dht6.write(dhtfile) +ariaconfig = ariaconfig.replace("/currentwd", str(CWD)) +try: + max_dl = getConfig("MAX_CONCURRENT_DOWNLOADS") +except KeyError: + max_dl = "4" +tracker_list = requests.get("https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all_aria2.txt").text +ariaconfig += f"\nmax-concurrent-downloads={max_dl}\nbt-tracker={tracker_list}" + +with open("aria.conf", "w+") as ariaconf: + ariaconf.write(ariaconfig) + +ARIA_CHILD_PROC = None +try: + ARIA_CHILD_PROC = subprocess.Popen(["aria2c", f"--conf-path={CWD}/aria.conf"]) +except FileNotFoundError: + LOGGER.error("Please install Aria2c, Exiting..") + sys.exit(0) +except OSError: + LOGGER.error("Aria2c Binary might have got damaged, Please Check and reinstall..") + sys.exit(0) +time.sleep(1) aria2 = aria2p.API( aria2p.Client( @@ -64,33 +106,124 @@ def getConfig(name: str): for line in lines: # LOGGER.info(line.split()) AUTHORIZED_CHATS.add(int(line.split()[0])) +try: + achats = getConfig('AUTHORIZED_CHATS') + achats = achats.split(" ") + for chats in achats: + AUTHORIZED_CHATS.add(int(chats)) +except: + pass + try: BOT_TOKEN = getConfig('BOT_TOKEN') parent_id = getConfig('GDRIVE_FOLDER_ID') DOWNLOAD_DIR = getConfig('DOWNLOAD_DIR') - if DOWNLOAD_DIR[-1] != '/' or DOWNLOAD_DIR[-1] != '\\': + if not DOWNLOAD_DIR.endswith("/"): DOWNLOAD_DIR = DOWNLOAD_DIR + '/' + if not os.path.exists(DOWNLOAD_DIR): + os.makedirs(DOWNLOAD_DIR, 0o777) DOWNLOAD_STATUS_UPDATE_INTERVAL = int(getConfig('DOWNLOAD_STATUS_UPDATE_INTERVAL')) OWNER_ID = int(getConfig('OWNER_ID')) AUTO_DELETE_MESSAGE_DURATION = int(getConfig('AUTO_DELETE_MESSAGE_DURATION')) - USER_SESSION_STRING = getConfig('USER_SESSION_STRING') TELEGRAM_API = getConfig('TELEGRAM_API') TELEGRAM_HASH = getConfig('TELEGRAM_HASH') except KeyError as e: LOGGER.error("One or more env variables missing! Exiting now") - exit(1) + sys.exit() + # exit() + +LOGGER.info("Generating USER_SESSION_STRING") +app = Client(':memory:', api_id=int(TELEGRAM_API), api_hash=TELEGRAM_HASH, bot_token=BOT_TOKEN) + +#Generate Telegraph Token +sname = ''.join(random.SystemRandom().choices(string.ascii_letters, k=8)) +LOGGER.info("Generating Telegraph Token using '" + sname + "' name") +telegraph = Telegraph() +telegraph.create_account(short_name=sname) +telegraph_token = telegraph.get_access_token() +LOGGER.info("Telegraph Token Generated: '" + telegraph_token + "'") try: - MEGA_API_KEY = getConfig('MEGA_API_KEY') + UPTOBOX_TOKEN = getConfig('UPTOBOX_TOKEN') except KeyError: - logging.warning('MEGA Api key not provided!') - MEGA_API_KEY = None + logging.warning('UPTOBOX_TOKEN not provided!') + UPTOBOX_TOKEN = None + +try: + MEGA_KEY = getConfig('MEGA_KEY') +except KeyError: + MEGA_KEY = None + LOGGER.info('MEGA API KEY NOT AVAILABLE') +MEGA_CHILD_PROC = None +if MEGA_KEY is not None: + try: + MEGA_CHILD_PROC = subprocess.Popen(["megasdkrest", "--apikey", MEGA_KEY]) + except FileNotFoundError: + LOGGER.error("Please install Megasdkrest Binary, Exiting..") + sys.exit(0) + except OSError: + LOGGER.error("Megasdkrest Binary might have got damaged, Please Check ..") + sys.exit(0) + time.sleep(3) + mega_client = MegaSdkRestClient('http://localhost:6090') + try: + MEGA_USERNAME = getConfig('MEGA_USERNAME') + MEGA_PASSWORD = getConfig('MEGA_PASSWORD') + if len(MEGA_USERNAME) > 0 and len(MEGA_PASSWORD) > 0: + try: + mega_client.login(MEGA_USERNAME, MEGA_PASSWORD) + except mega_err.MegaSdkRestClientException as e: + logging.error(e.message['message']) + exit(0) + else: + LOGGER.info("Mega API KEY provided but credentials not provided. Starting mega in anonymous mode!") + MEGA_USERNAME = None + MEGA_PASSWORD = None + except KeyError: + LOGGER.info("Mega API KEY provided but credentials not provided. Starting mega in anonymous mode!") + MEGA_USERNAME = None + MEGA_PASSWORD = None +else: + MEGA_USERNAME = None + MEGA_PASSWORD = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: INDEX_URL = None except KeyError: INDEX_URL = None +try: + BUTTON_THREE_NAME = getConfig('BUTTON_THREE_NAME') + BUTTON_THREE_URL = getConfig('BUTTON_THREE_URL') + if len(BUTTON_THREE_NAME) == 0 or len(BUTTON_THREE_URL) == 0: + raise KeyError +except KeyError: + BUTTON_THREE_NAME = None + BUTTON_THREE_URL = None +try: + BUTTON_FOUR_NAME = getConfig('BUTTON_FOUR_NAME') + BUTTON_FOUR_URL = getConfig('BUTTON_FOUR_URL') + if len(BUTTON_FOUR_NAME) == 0 or len(BUTTON_FOUR_URL) == 0: + raise KeyError +except KeyError: + BUTTON_FOUR_NAME = None + BUTTON_FOUR_URL = None +try: + BUTTON_FIVE_NAME = getConfig('BUTTON_FIVE_NAME') + BUTTON_FIVE_URL = getConfig('BUTTON_FIVE_URL') + if len(BUTTON_FIVE_NAME) == 0 or len(BUTTON_FIVE_URL) == 0: + raise KeyError +except KeyError: + BUTTON_FIVE_NAME = None + BUTTON_FIVE_URL = None +try: + STOP_DUPLICATE_MIRROR = getConfig('STOP_DUPLICATE_MIRROR') + if STOP_DUPLICATE_MIRROR.lower() == 'true': + STOP_DUPLICATE_MIRROR = True + else: + STOP_DUPLICATE_MIRROR = False +except KeyError: + STOP_DUPLICATE_MIRROR = False try: IS_TEAM_DRIVE = getConfig('IS_TEAM_DRIVE') if IS_TEAM_DRIVE.lower() == 'true': @@ -109,6 +242,24 @@ def getConfig(name: str): except KeyError: USE_SERVICE_ACCOUNTS = False -updater = tg.Updater(token=BOT_TOKEN,use_context=True) +try: + BLOCK_MEGA_LINKS = getConfig('BLOCK_MEGA_LINKS') + if BLOCK_MEGA_LINKS.lower() == 'true': + BLOCK_MEGA_LINKS = True + else: + BLOCK_MEGA_LINKS = False +except KeyError: + BLOCK_MEGA_LINKS = False + +try: + SHORTENER = getConfig('SHORTENER') + SHORTENER_API = getConfig('SHORTENER_API') + if len(SHORTENER) == 0 or len(SHORTENER_API) == 0: + raise KeyError +except KeyError: + SHORTENER = None + SHORTENER_API = None + +updater = tg.Updater(token=BOT_TOKEN) bot = updater.bot dispatcher = updater.dispatcher diff --git a/bot/__main__.py b/bot/__main__.py index 34df995b5..5242d529d 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,35 +1,46 @@ -import shutil +import os +import shutil, psutil import signal -import pickle -from os import execl, path, remove from sys import executable +import time -from telegram.ext import CommandHandler, run_async -from bot import dispatcher, updater, botStartTime +from telegram.ext import CommandHandler +from bot import bot, dispatcher, updater, botStartTime from bot.helper.ext_utils import fs_utils from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.message_utils import * from .helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time from .helper.telegram_helper.filters import CustomFilters -from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone, watch +from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone, watch, delete, speedtest, count + +from pyrogram import idle +from bot import app -@run_async def stats(update, context): - currentTime = get_readable_time((time.time() - botStartTime)) + currentTime = get_readable_time(time.time() - botStartTime) total, used, free = shutil.disk_usage('.') total = get_readable_file_size(total) used = get_readable_file_size(used) free = get_readable_file_size(free) - stats = f'Bot Uptime: {currentTime}\n' \ - f'Total disk space: {total}\n' \ - f'Used: {used}\n' \ - f'Free: {free}' + sent = get_readable_file_size(psutil.net_io_counters().bytes_sent) + recv = get_readable_file_size(psutil.net_io_counters().bytes_recv) + cpuUsage = psutil.cpu_percent(interval=0.5) + memory = psutil.virtual_memory().percent + disk = psutil.disk_usage('/').percent + stats = f'Bot Uptime:- {currentTime}\n' \ + f'Total Disk Space:- {total}\n' \ + f'Used:- {used} ' \ + f'Free:- {free}\n\n' \ + f'Data Usage\nUp:- {sent}\n' \ + f'Down:- {recv}\n\n' \ + f'CPU: {cpuUsage}% ' \ + f'RAM: {memory}% ' \ + f'Disk: {disk}%' sendMessage(stats, context.bot, update) -@run_async def start(update, context): start_string = f''' This is a bot which can mirror all your links to Google drive! @@ -38,17 +49,16 @@ def start(update, context): sendMessage(start_string, context.bot, update) -@run_async def restart(update, context): restart_message = sendMessage("Restarting, Please wait!", context.bot, update) - # Save restart message object in order to reply to it after restarting + # Save restart message ID and chat ID in order to edit it after restarting + with open(".restartmsg", "w") as f: + f.truncate(0) + f.write(f"{restart_message.chat.id}\n{restart_message.message_id}\n") fs_utils.clean_all() - with open('restart.pickle', 'wb') as status: - pickle.dump(restart_message, status) - execl(executable, executable, "-m", "bot") + os.execl(executable, executable, "-m", "bot") -@run_async def ping(update, context): start_time = int(round(time.time() * 1000)) reply = sendMessage("Starting Ping", context.bot, update) @@ -56,23 +66,23 @@ def ping(update, context): editMessage(f'{end_time - start_time} ms', reply) -@run_async def log(update, context): sendLogFile(context.bot, update) -@run_async def bot_help(update, context): help_string = f''' /{BotCommands.HelpCommand}: To get this message -/{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive +/{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive.\n /{BotCommands.UnzipMirrorCommand} [download_url][magnet_link] : starts mirroring and if downloaded file is any archive , extracts it to google drive +/{BotCommands.CountCommand}: Count files/folders of G-Drive Links + /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download -/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl +/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl. Click /{BotCommands.WatchCommand} for more help. /{BotCommands.TarWatchCommand} [youtube-dl supported link]: Mirror through youtube-dl and tar before uploading @@ -88,6 +98,8 @@ def bot_help(update, context): /{BotCommands.LogCommand}: Get a log file of the bot. Handy for getting crash reports +/{BotCommands.SpeedCommand} : Check Internet Speed Of The Host + ''' sendMessage(help_string, context.bot, update) @@ -95,23 +107,23 @@ def bot_help(update, context): def main(): fs_utils.start_cleanup() # Check if the bot is restarting - if path.exists('restart.pickle'): - with open('restart.pickle', 'rb') as status: - restart_message = pickle.load(status) - restart_message.edit_text("Restarted Successfully!") - remove('restart.pickle') + if os.path.isfile(".restartmsg"): + with open(".restartmsg") as f: + chat_id, msg_id = map(int, f) + bot.edit_message_text("Restarted successfully!", chat_id, msg_id) + os.remove(".restartmsg") start_handler = CommandHandler(BotCommands.StartCommand, start, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) ping_handler = CommandHandler(BotCommands.PingCommand, ping, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) restart_handler = CommandHandler(BotCommands.RestartCommand, restart, - filters=CustomFilters.owner_filter) + filters=CustomFilters.owner_filter| CustomFilters.authorized_user, run_async=True) help_handler = CommandHandler(BotCommands.HelpCommand, - bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) stats_handler = CommandHandler(BotCommands.StatsCommand, - stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) - log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter) + stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) + log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter, run_async=True) dispatcher.add_handler(start_handler) dispatcher.add_handler(ping_handler) dispatcher.add_handler(restart_handler) @@ -122,5 +134,6 @@ def main(): LOGGER.info("Bot Started!") signal.signal(signal.SIGINT, fs_utils.exit_clean_up) - +app.start() main() +idle() diff --git a/bot/data/aria.conf b/bot/data/aria.conf new file mode 100644 index 000000000..b3a6a2ce5 --- /dev/null +++ b/bot/data/aria.conf @@ -0,0 +1,43 @@ +# RPC +enable-rpc=true +rpc-listen-all=true +rpc-listen-port=6800 +rpc-max-request-size=1024M + +# Connection +max-connection-per-server=14 +check-certificate=false +min-split-size=10M +max-overall-download-limit=0 +disable-ipv6=true +split=10 + +# Aria +# daemon=true +quiet=true +allow-overwrite=true +file-allocation=prealloc + +#BT +listen-port=51413 +bt-max-peers=0 +bt-request-peer-speed-limit=0 +max-overall-upload-limit=0 +seed-time=0.01 +bt-tracker-connect-timeout=600 +follow-torrent=mem +bt-force-encryption=true +bt-stop-timeout=600 +user-agent=qBittorrent/4.3.5 +peer-agent=qBittorrent/4.3.5 +peer-id-prefix=-qB4350- + +#DHT +dht-listen-port=51513 +enable-dht=true +enable-dht6=false +dht-file-path=/currentwd/dht.dat +dht-file-path6=/currentwd/dht6.dat +dht-entry-point=dht.transmissionbt.com:6881 +dht-entry-point6=dht.transmissionbt.com:6881 + diff --git a/bot/data/dht.dat b/bot/data/dht.dat new file mode 100644 index 000000000..a834b7771 Binary files /dev/null and b/bot/data/dht.dat differ diff --git a/bot/data/dht6.dat b/bot/data/dht6.dat new file mode 100644 index 000000000..36d34dbd8 Binary files /dev/null and b/bot/data/dht6.dat differ diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index a015fb79f..2de4b3ca8 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -3,6 +3,7 @@ import threading import time +from bot.helper.telegram_helper.bot_commands import BotCommands from bot import download_dict, download_dict_lock LOGGER = logging.getLogger(__name__) @@ -16,14 +17,14 @@ class MirrorStatus: STATUS_UPLOADING = "Uploading" STATUS_DOWNLOADING = "Downloading" STATUS_WAITING = "Queued" - STATUS_FAILED = "Failed. Cleaning download" - STATUS_CANCELLED = "Cancelled" + STATUS_FAILED = "Failed.Cleaning download" + STATUS_CANCELLED = "Cancelled " STATUS_ARCHIVING = "Archiving" STATUS_EXTRACTING = "Extracting" PROGRESS_MAX_SIZE = 100 // 8 -PROGRESS_INCOMPLETE = ['▏', '▎', '▍', '▌', '▋', '▊', '▉'] +PROGRESS_INCOMPLETE = ['█', '█', '█', '█', '█', '█', '█'] SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] @@ -62,7 +63,9 @@ def get_readable_file_size(size_in_bytes) -> str: def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): - if dl.status() != MirrorStatus.STATUS_UPLOADING and dl.status() != MirrorStatus.STATUS_ARCHIVING: + status = dl.status() + if status != MirrorStatus.STATUS_UPLOADING and status != MirrorStatus.STATUS_ARCHIVING \ + and status != MirrorStatus.STATUS_EXTRACTING: if dl.gid() == gid: return dl return None @@ -81,7 +84,7 @@ def get_progress_bar_string(status): p_str = '█' * cFull if cPart >= 0: p_str += PROGRESS_INCOMPLETE[cPart] - p_str += ' ' * (PROGRESS_MAX_SIZE - cFull) + p_str += '░' * (PROGRESS_MAX_SIZE - cFull) p_str = f"[{p_str}]" return p_str @@ -90,17 +93,23 @@ def get_readable_message(): with download_dict_lock: msg = "" for download in list(download_dict.values()): - msg += f"{download.name()} - " - msg += download.status() + msg += f"Name:- {download.name()}" + msg += f"\nStatus:- {download.status()}" if download.status() != MirrorStatus.STATUS_ARCHIVING and download.status() != MirrorStatus.STATUS_EXTRACTING: - msg += f"\n{get_progress_bar_string(download)} {download.progress()} of " \ - f"{download.size()}" \ - f" at {download.speed()}, ETA: {download.eta()} " + msg += f"\n{get_progress_bar_string(download)} {download.progress()}" + if download.status() == MirrorStatus.STATUS_DOWNLOADING: + msg += f"\nDownloaded:- {get_readable_file_size(download.processed_bytes())} of {download.size()}" + else: + msg += f"\nUploaded:- {get_readable_file_size(download.processed_bytes())} of {download.size()}" + msg += f"\nSpeed:- {download.speed()}, \nETA:- {download.eta()} " + # if hasattr(download, 'is_torrent'): + try: + msg += f"\nInfo: Seeders:- {download.aria_download().num_seeders}" \ + f" & Peers:- {download.aria_download().connections}" + except: + pass if download.status() == MirrorStatus.STATUS_DOWNLOADING: - if hasattr(download, 'is_torrent'): - msg += f"| P: {download.aria_download().connections} " \ - f"| S: {download.aria_download().num_seeders}" - msg += f"\nGID: {download.gid()}" + msg += f"\nTo Stop:- /{BotCommands.CancelMirror} {download.gid()}" msg += "\n\n" return msg @@ -137,15 +146,30 @@ def is_magnet(url: str): return True return False +def is_gdrive_link(url: str): + return "drive.google.com" in url + def is_mega_link(url: str): return "mega.nz" in url +def get_mega_link_type(url: str): + if "folder" in url: + return "folder" + elif "file" in url: + return "file" + elif "/#F!" in url: + return "folder" + return "file" + + def new_thread(fn): """To use as decorator to make a function call threaded. Needs import from threading import Thread""" + def wrapper(*args, **kwargs): thread = threading.Thread(target=fn, args=args, kwargs=kwargs) thread.start() return thread + return wrapper diff --git a/bot/helper/ext_utils/exceptions.py b/bot/helper/ext_utils/exceptions.py index 25ff87fa6..a2f600c27 100644 --- a/bot/helper/ext_utils/exceptions.py +++ b/bot/helper/ext_utils/exceptions.py @@ -1,2 +1,8 @@ class DirectDownloadLinkException(Exception): + """Not method found for extracting direct download link from the http link""" + pass + + +class NotSupportedExtractionArchive(Exception): + """The archive format use is trying to extract is not supported""" pass diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 90951d60a..d1bc9bd22 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -1,10 +1,11 @@ import sys -from bot import aria2, LOGGER, DOWNLOAD_DIR +from bot import aria2, LOGGER, DOWNLOAD_DIR, ARIA_CHILD_PROC, MEGA_CHILD_PROC import shutil import os import pathlib import magic import tarfile +from .exceptions import NotSupportedExtractionArchive def clean_download(path: str): @@ -22,16 +23,23 @@ def start_cleanup(): def clean_all(): aria2.remove_all(True) - shutil.rmtree(DOWNLOAD_DIR) + try: + shutil.rmtree(DOWNLOAD_DIR) + except FileNotFoundError: + pass def exit_clean_up(signal, frame): try: LOGGER.info("Please wait, while we clean up the downloads and stop running downloads") clean_all() + ARIA_CHILD_PROC.kill() + MEGA_CHILD_PROC.kill() sys.exit(0) except KeyboardInterrupt: LOGGER.warning("Force Exiting before the cleanup finishes!") + ARIA_CHILD_PROC.kill() + MEGA_CHILD_PROC.kill() sys.exit(1) @@ -60,25 +68,77 @@ def get_base_name(orig_path: str): if orig_path.endswith(".tar.bz2"): return orig_path.replace(".tar.bz2", "") elif orig_path.endswith(".tar.gz"): - return orig_path.replace(".tar.gz","") + return orig_path.replace(".tar.gz", "") elif orig_path.endswith(".bz2"): - return orig_path.replace(".bz2","") + return orig_path.replace(".bz2", "") elif orig_path.endswith(".gz"): - return orig_path.replace(".gz","") + return orig_path.replace(".gz", "") elif orig_path.endswith(".tar"): - return orig_path.replace(".tar","") + return orig_path.replace(".tar", "") elif orig_path.endswith(".tbz2"): - return orig_path.replace("tbz2","") + return orig_path.replace("tbz2", "") elif orig_path.endswith(".tgz"): - return orig_path.replace(".tgz","") + return orig_path.replace(".tgz", "") elif orig_path.endswith(".zip"): - return orig_path.replace(".zip","") + return orig_path.replace(".zip", "") + elif orig_path.endswith(".7z"): + return orig_path.replace(".7z", "") elif orig_path.endswith(".Z"): - return orig_path.replace(".Z","") + return orig_path.replace(".Z", "") elif orig_path.endswith(".rar"): - return orig_path.replace(".rar","") + return orig_path.replace(".rar", "") + elif orig_path.endswith(".iso"): + return orig_path.replace(".iso", "") + elif orig_path.endswith(".wim"): + return orig_path.replace(".wim", "") + elif orig_path.endswith(".cab"): + return orig_path.replace(".cab", "") + elif orig_path.endswith(".apm"): + return orig_path.replace(".apm", "") + elif orig_path.endswith(".arj"): + return orig_path.replace(".arj", "") + elif orig_path.endswith(".chm"): + return orig_path.replace(".chm", "") + elif orig_path.endswith(".cpio"): + return orig_path.replace(".cpio", "") + elif orig_path.endswith(".cramfs"): + return orig_path.replace(".cramfs", "") + elif orig_path.endswith(".deb"): + return orig_path.replace(".deb", "") + elif orig_path.endswith(".dmg"): + return orig_path.replace(".dmg", "") + elif orig_path.endswith(".fat"): + return orig_path.replace(".fat", "") + elif orig_path.endswith(".hfs"): + return orig_path.replace(".hfs", "") + elif orig_path.endswith(".lzh"): + return orig_path.replace(".lzh", "") + elif orig_path.endswith(".lzma"): + return orig_path.replace(".lzma", "") + elif orig_path.endswith(".lzma2"): + return orig_path.replace(".lzma2", "") + elif orig_path.endswith(".mbr"): + return orig_path.replace(".mbr", "") + elif orig_path.endswith(".msi"): + return orig_path.replace(".msi", "") + elif orig_path.endswith(".mslz"): + return orig_path.replace(".mslz", "") + elif orig_path.endswith(".nsis"): + return orig_path.replace(".nsis", "") + elif orig_path.endswith(".ntfs"): + return orig_path.replace(".ntfs", "") + elif orig_path.endswith(".rpm"): + return orig_path.replace(".rpm", "") + elif orig_path.endswith(".squashfs"): + return orig_path.replace(".squashfs", "") + elif orig_path.endswith(".udf"): + return orig_path.replace(".udf", "") + elif orig_path.endswith(".vhd"): + return orig_path.replace(".vhd", "") + elif orig_path.endswith(".xar"): + return orig_path.replace(".xar", "") else: - return "unsupported" + raise NotSupportedExtractionArchive('File format not supported for extraction') def get_mime_type(file_path): diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index e30a0a9d9..defdca6c4 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -1,4 +1,5 @@ -from bot import aria2, download_dict_lock +from bot import aria2, download_dict_lock, STOP_DUPLICATE_MIRROR +from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper from bot.helper.ext_utils.bot_utils import * from .download_helper import DownloadHelper from bot.helper.mirror_utils.status_utils.aria_download_status import AriaDownloadStatus @@ -15,18 +16,36 @@ def __init__(self): @new_thread def __onDownloadStarted(self, api, gid): + sleep(1) LOGGER.info(f"onDownloadStart: {gid}") + dl = getDownloadByGid(gid) + download = api.get_download(gid) + self.name = download.name + sname = download.name + if STOP_DUPLICATE_MIRROR: + if dl.getListener().isTar == True: + sname = sname + ".tar" + if dl.getListener().extract == True: + smsg = None + else: + gdrive = GoogleDriveHelper(None) + smsg, button = gdrive.drive_list(sname) + if smsg: + dl.getListener().onDownloadError(f'File is already available in drive.This download has been stopped.\n\n') + sendMarkup("Here are the search results:", dl.getListener().bot, dl.getListener().update, button) + aria2.remove([download]) + return update_all_messages() - def __onDownloadComplete(self, api: API, gid): LOGGER.info(f"onDownloadComplete: {gid}") dl = getDownloadByGid(gid) download = api.get_download(gid) if download.followed_by_ids: new_gid = download.followed_by_ids[0] + new_download = api.get_download(new_gid) with download_dict_lock: - download_dict[dl.uid()] = AriaDownloadStatus(new_gid,dl.getListener()) - if download.is_torrent: + download_dict[dl.uid()] = AriaDownloadStatus(new_gid, dl.getListener()) + if new_download.is_torrent: download_dict[dl.uid()].is_torrent = True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {new_gid}') @@ -43,7 +62,7 @@ def __onDownloadPause(self, api, gid): def __onDownloadStopped(self, api, gid): LOGGER.info(f"onDownloadStop: {gid}") dl = getDownloadByGid(gid) - if dl: dl.getListener().onDownloadError('Download stopped by user!') + if dl: dl.getListener().onDownloadError('Your torrent has no seeds.Download stopped automatically.') @new_thread def __onDownloadError(self, api, gid): @@ -63,11 +82,11 @@ def start_listener(self): on_download_complete=self.__onDownloadComplete) - def add_download(self, link: str, path,listener): + def add_download(self, link: str, path, listener, filename): if is_magnet(link): - download = aria2.add_magnet(link, {'dir': path}) + download = aria2.add_magnet(link, {'dir': path, 'out': filename}) else: - download = aria2.add_uris([link], {'dir': path}) + download = aria2.add_uris([link], {'dir': path, 'out': filename}) if download.error_message: #no need to proceed further at this point listener.onDownloadError(download.error_message) return diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py index 84ea43a5b..b00728517 100644 --- a/bot/helper/mirror_utils/download_utils/direct_link_generator.py +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -10,12 +10,20 @@ import json import re +import math import urllib.parse from os import popen from random import choice +from urllib.parse import urlparse +import lk21 import requests +import logging +from bot import UPTOBOX_TOKEN from bs4 import BeautifulSoup +from lk21.extractors.bypasser import Bypass +from base64 import standard_b64encode +from js2py import EvalJs from bot.helper.ext_utils.exceptions import DirectDownloadLinkException @@ -32,38 +40,45 @@ def direct_link_generator(link: str): return cm_ru(link) elif 'mediafire.com' in link: return mediafire(link) + elif 'uptobox.com' in link: + return uptobox(link) elif 'osdn.net' in link: return osdn(link) elif 'github.com' in link: return github(link) + elif 'fembed.com' in link: + return fembed(link) + elif 'femax20.com' in link: + return fembed(link) + elif 'feurl.com' in link: + return fembed(link) else: raise DirectDownloadLinkException(f'No Direct link function found for {link}') def zippy_share(url: str) -> str: - """ ZippyShare direct links generator - Based on https://github.com/LameLemon/ziggy""" - dl_url = '' + link = re.findall("https:/.(.*?).zippyshare", url)[0] + response_content = (requests.get(url)).content + bs_obj = BeautifulSoup(response_content, "lxml") + try: - link = re.findall(r'\bhttps?://.*zippyshare\.com\S+', url)[0] - except IndexError: - raise DirectDownloadLinkException("`No ZippyShare links found`\n") - session = requests.Session() - base_url = re.search('http.+.com', link).group() - response = session.get(link) - page_soup = BeautifulSoup(response.content, "lxml") - scripts = page_soup.find_all("script", {"type": "text/javascript"}) - for script in scripts: - if "getElementById('dlbutton')" in script.text: - url_raw = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', - script.text).group('url') - math = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', - script.text).group('math') - dl_url = url_raw.replace(math, '"' + str(eval(math)) + '"') - break - dl_url = base_url + eval(dl_url) - name = urllib.parse.unquote(dl_url.split('/')[-1]) - return dl_url + js_script = bs_obj.find("div", {"class": "center",}).find_all( + "script" + )[1] + except: + js_script = bs_obj.find("div", {"class": "right",}).find_all( + "script" + )[0] + + js_content = re.findall(r'\.href.=."/(.*?)";', str(js_script)) + js_content = 'var x = "/' + js_content[0] + '"' + + evaljs = EvalJs() + setattr(evaljs, "x", None) + evaljs.execute(js_content) + js_content = getattr(evaljs, "x") + + return f"https://{link}.zippyshare.com{js_content}" def yandex_disk(url: str) -> str: @@ -113,6 +128,44 @@ def mediafire(url: str) -> str: return dl_url +def uptobox(url: str) -> str: + try: + link = re.findall(r'\bhttps?://.*uptobox\.com\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No Uptobox links found`\n") + if UPTOBOX_TOKEN is None: + logging.error('UPTOBOX_TOKEN not provided!') + else: + check = 'https://uptobox.com/api/user/me?token=%s' % (UPTOBOX_TOKEN) + request = requests.get(check) + info = request.json() + premium = info["data"]["premium"] + try: + link = re.findall(r'\bhttp?://.*uptobox\.com/dl\S+', url)[0] + logging.info('Uptobox direct link') + dl_url = url + except: + if premium == 1: + file_id = re.findall(r'\bhttps?://.*uptobox\.com/(\w+)', url)[0] + file_link = 'https://uptobox.com/api/link?token=%s&file_code=%s' % (UPTOBOX_TOKEN, file_id) + req = requests.get(file_link) + result = req.json() + dl_url = result['data']['dlLink'] + else: + file_id = re.findall(r'\bhttps?://.*uptobox\.com/(\w+)', url)[0] + file_link = 'https://uptobox.com/api/link?token=%s&file_code=%s' % (UPTOBOX_TOKEN, file_id) + req = requests.get(file_link) + result = req.json() + waiting_time = result["data"]["waiting"] + 1 + waiting_token = result["data"]["waitingToken"] + _countdown(waiting_time) + file_link = 'https://uptobox.com/api/link?token=%s&file_code=%s&waitingToken=%s' % (UPTOBOX_TOKEN, file_id, waiting_token) + req = requests.get(file_link) + result = req.json() + dl_url = result['data']['dlLink'] + return dl_url + + def osdn(url: str) -> str: """ OSDN direct links generator """ osdn_link = 'https://osdn.net' @@ -157,3 +210,14 @@ def useragent(): 'lxml').findAll('td', {'class': 'useragent'}) user_agent = choice(useragents) return user_agent.text + +def fembed(link: str) -> str: + """ Fembed direct link generator + Based on https://github.com/breakdowns/slam-mirrorbot """ + bypasser = lk21.Bypass() + dl_url=bypasser.bypass_fembed(link) + lst_link = [] + count = len(dl_url) + for i in dl_url: + lst_link.append(dl_url[i]) + return lst_link[count-1] diff --git a/bot/helper/mirror_utils/download_utils/download_helper.py b/bot/helper/mirror_utils/download_utils/download_helper.py index ebcca1444..b8960e250 100644 --- a/bot/helper/mirror_utils/download_utils/download_helper.py +++ b/bot/helper/mirror_utils/download_utils/download_helper.py @@ -9,8 +9,8 @@ def __init__(self): class DownloadHelper: def __init__(self): - self.name = '' # Name of the download; empty string if no download has been started - self.size = 0.0 # Size of the download + self.__name = '' # Name of the download; empty string if no download has been started + self.__size = 0.0 # Size of the download self.downloaded_bytes = 0.0 # Bytes downloaded self.speed = 0.0 # Download speed in bytes per second self.progress = 0.0 diff --git a/bot/helper/mirror_utils/download_utils/mega_download.py b/bot/helper/mirror_utils/download_utils/mega_download.py new file mode 100644 index 000000000..cc62cc0f5 --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/mega_download.py @@ -0,0 +1,108 @@ +import threading +from bot import LOGGER, download_dict, download_dict_lock +from .download_helper import DownloadHelper +from ..status_utils.mega_status import MegaDownloadStatus +from megasdkrestclient import MegaSdkRestClient, constants +from bot.helper.ext_utils.bot_utils import setInterval +from pathlib import Path + + +class MegaDownloader: + POLLING_INTERVAL = 2 + + def __init__(self, listener): + super().__init__() + self.__listener = listener + self.__name = "" + self.__gid = '' + self.__resource_lock = threading.Lock() + self.__mega_client = MegaSdkRestClient('http://localhost:6090') + self.__periodic = None + self.__downloaded_bytes = 0 + self.__progress = 0 + self.__size = 0 + + @property + def progress(self): + with self.__resource_lock: + return self.__progress + + @property + def downloaded_bytes(self): + with self.__resource_lock: + return self.__downloaded_bytes + + @property + def size(self): + with self.__resource_lock: + return self.__size + + @property + def gid(self): + with self.__resource_lock: + return self.__gid + + @property + def name(self): + with self.__resource_lock: + return self.__name + + @property + def download_speed(self): + if self.gid is not None: + return self.__mega_client.getDownloadInfo(self.gid)['speed'] + + def __onDownloadStart(self, name, size, gid): + self.__periodic = setInterval(self.POLLING_INTERVAL, self.__onInterval) + with download_dict_lock: + download_dict[self.__listener.uid] = MegaDownloadStatus(self, self.__listener) + with self.__resource_lock: + self.__name = name + self.__size = size + self.__gid = gid + self.__listener.onDownloadStarted() + + def __onInterval(self): + dlInfo = self.__mega_client.getDownloadInfo(self.gid) + if (dlInfo['state'] == constants.State.TYPE_STATE_COMPLETED or dlInfo[ + 'state'] == constants.State.TYPE_STATE_CANCELED or dlInfo[ + 'state'] == constants.State.TYPE_STATE_FAILED) and self.__periodic is not None: + self.__periodic.cancel() + if dlInfo['state'] == constants.State.TYPE_STATE_COMPLETED: + self.__onDownloadComplete() + return + if dlInfo['state'] == constants.State.TYPE_STATE_CANCELED: + self.__onDownloadError('Cancelled by user') + return + if dlInfo['state'] == constants.State.TYPE_STATE_FAILED: + self.__onDownloadError(dlInfo['error_string']) + return + self.__onDownloadProgress(dlInfo['completed_length'], dlInfo['total_length']) + + def __onDownloadProgress(self, current, total): + with self.__resource_lock: + self.__downloaded_bytes = current + try: + self.__progress = current / total * 100 + except ZeroDivisionError: + self.__progress = 0 + + def __onDownloadError(self, error): + self.__listener.onDownloadError(error) + + def __onDownloadComplete(self): + self.__listener.onDownloadComplete() + + def add_download(self, link, path): + Path(path).mkdir(parents=True, exist_ok=True) + dl = self.__mega_client.addDl(link, path) + gid = dl['gid'] + info = self.__mega_client.getDownloadInfo(gid) + file_name = info['name'] + file_size = info['total_length'] + self.__onDownloadStart(file_name, file_size, gid) + LOGGER.info(f'Started mega download with gid: {gid}') + + def cancel_download(self): + LOGGER.info(f'Cancelling download on user request: {self.gid}') + self.__mega_client.cancelDl(self.gid) \ No newline at end of file diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py deleted file mode 100644 index 4714b113c..000000000 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ /dev/null @@ -1,132 +0,0 @@ -from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict -import threading -from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) -from bot.helper.telegram_helper.message_utils import update_all_messages -import os -from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus - -class MegaAppListener(MegaListener): - - def __init__(self, continue_event: threading.Event, listener): - self.continue_event = continue_event - self.node = None - self.listener = listener - self.uid = listener.uid - self.__bytes_transferred = 0 - self.is_cancelled = False - self.__speed = 0 - self.__name = '' - self.__size = 0 - self.error = None - self.gid = "" - super(MegaAppListener, self).__init__() - - @property - def speed(self): - """Returns speed of the download in bytes/second""" - return self.__speed - - @property - def name(self): - """Returns name of the download""" - return self.__name - - def setValues(self,name,size,gid): - self.__name = name - self.__size = size - self.gid = gid - - @property - def size(self): - """Size of download in bytes""" - return self.__size - - @property - def downloaded_bytes(self): - return self.__bytes_transferred - - def onRequestStart(self, api, request): - LOGGER.info('Request start ({})'.format(request)) - - def onRequestFinish(self, api, request, error): - LOGGER.info('Mega Request finished ({}); Result: {}' - .format(request, error)) - - request_type = request.getType() - if request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: - self.node = request.getPublicMegaNode() - if request_type == MegaRequest.TYPE_LOGIN: - LOGGER.info("Fetching Nodes.") - api.fetchNodes() - if request_type == MegaRequest.TYPE_FETCH_NODES: - LOGGER.info("Fetching Root Node.") - self.node = api.getRootNode() - if request_type != MegaRequest.TYPE_LOGIN: - self.continue_event.set() - - def onRequestTemporaryError(self, api, request, error: MegaError): - self.listener.onDownloadError(error.toString()) - self.error = error.toString() - self.continue_event.set() - - def onTransferStart(self, api: MegaApi, transfer: MegaTransfer): - LOGGER.info(f"Transfer Started: {transfer.getFileName()}") - - def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): - if self.is_cancelled: - api.cancelTransfer(transfer,None) - self.__speed = transfer.getSpeed() - self.__bytes_transferred = transfer.getTransferredBytes() - - def onTransferFinish(self, api: MegaApi, transfer : MegaTransfer, error): - try: - LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') - if str(error) != "No error" and self.is_cancelled: - self.is_cancelled = False - return self.listener.onDownloadError(error.toString()) - if transfer.isFolderTransfer() and transfer.isFinished() and not self.is_cancelled or transfer.getFileName() == self.name and not self.is_cancelled: - self.listener.onDownloadComplete() - except Exception as e: - LOGGER.error(e) - - def onTransferTemporaryError(self, api, transfer, error): - LOGGER.info(f'Mega download error in file {transfer} {transfer.getFileName()}: {error}') - self.listener.onDownloadError(error.toString()) - self.error = error.toString() - self.continue_event.set() - - def cancel_download(self): - self.is_cancelled = True - -class AsyncExecutor(object): - - def __init__(self): - self.continue_event = threading.Event() - - def do(self, function, args): - self.continue_event.clear() - function(*args) - self.continue_event.wait() - -class MegaDownloadHelper: - def __init__(self): - pass - - def add_download(self, mega_link: str, path: str, listener): - executor = AsyncExecutor() - api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') - mega_listener = MegaAppListener(executor.continue_event, listener) - os.makedirs(path) - api.addListener(mega_listener) - executor.do(api.getPublicNode, (mega_link,)) - node = mega_listener.node - if node is None: - executor.do(api.loginToFolder, (mega_link,)) - node = mega_listener.node - if mega_listener.error is not None: - return listener.onDownloadError(str(mega_listener.error)) - mega_listener.setValues(node.getName(),api.getSize(node),mega_link.split("!",1)[-1].split("!",1)[0]) - with download_dict_lock: - download_dict[listener.uid] = MegaDownloadStatus(mega_listener,listener) - threading.Thread(target=executor.do,args=(api.startDownload,(node,path))).start() - update_all_messages() diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index b35f7f3dd..59a926d9f 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -2,10 +2,7 @@ import threading import time -from pyrogram import Client - -from bot import LOGGER, download_dict, download_dict_lock, TELEGRAM_API, \ - TELEGRAM_HASH, USER_SESSION_STRING +from bot import LOGGER, download_dict, download_dict_lock, app from .download_helper import DownloadHelper from ..status_utils.telegram_download_status import TelegramDownloadStatus @@ -23,10 +20,7 @@ def __init__(self, listener): self.__name = "" self.__gid = '' self.__start_time = time.time() - self.__user_bot = Client(api_id=TELEGRAM_API, - api_hash=TELEGRAM_HASH, - session_name=USER_SESSION_STRING) - self.__user_bot.start() + self.__user_bot = app self.__is_cancelled = False @property @@ -84,7 +78,7 @@ def __download(self, message, path): if not self.__is_cancelled: self.__onDownloadError('Internal error occurred') - def add_download(self, message, path): + def add_download(self, message, path, filename): _message = self.__user_bot.get_messages(message.chat.id, message.message_id) media = None media_array = [_message.document, _message.video, _message.audio] @@ -96,9 +90,13 @@ def add_download(self, message, path): with global_lock: # For avoiding locking the thread lock for long time unnecessarily download = media.file_id not in GLOBAL_GID - + if filename == "": + name = media.file_name + else: + name = filename + path = path + name if download: - self.__onDownloadStart(media.file_name, media.file_size, media.file_id) + self.__onDownloadStart(name, media.file_size, media.file_id) LOGGER.info(f'Downloading telegram file with id: {media.file_id}') threading.Thread(target=self.__download, args=(_message, path)).start() else: diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index ca07ec1bf..1bf3bb747 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -1,6 +1,6 @@ from .download_helper import DownloadHelper import time -from youtube_dl import YoutubeDL, DownloadError +from yt_dlp import YoutubeDL, DownloadError from bot import download_dict_lock, download_dict from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus import logging @@ -19,7 +19,10 @@ def debug(self, msg): # Hack to fix changing changing extension match = re.search(r'.ffmpeg..Merging formats into..(.*?).$', msg) if match and not self.obj.is_playlist: - self.obj.name = match.group(1) + newname = match.group(1) + newname = newname.split("/") + newname = newname[-1] + self.obj.name = newname @staticmethod def warning(msg): @@ -33,15 +36,14 @@ def error(msg): class YoutubeDLHelper(DownloadHelper): def __init__(self, listener): super().__init__() - self.__name = "" + self.name = "" self.__start_time = time.time() self.__listener = listener self.__gid = "" self.opts = { 'progress_hooks': [self.__onDownloadProgress], 'logger': MyLogger(self), - 'usenetrc': True, - 'format': "best/bestvideo+bestaudio" + 'usenetrc': True } self.__download_speed = 0 self.download_speed_readable = '' @@ -72,10 +74,14 @@ def __onDownloadProgress(self, d): elif d['status'] == "downloading": with self.__resource_lock: self.__download_speed = d['speed'] + try: + tbyte = d['total_bytes'] + except KeyError: + tbyte = d['total_bytes_estimate'] if self.is_playlist: - progress = d['downloaded_bytes'] / d['total_bytes'] + progress = d['downloaded_bytes'] / tbyte chunk_size = d['downloaded_bytes'] - self.last_downloaded - self.last_downloaded = d['total_bytes'] * progress + self.last_downloaded = tbyte * progress self.downloaded_bytes += chunk_size try: self.progress = (self.downloaded_bytes / self.size) * 100 @@ -95,14 +101,20 @@ def __onDownloadComplete(self): def onDownloadError(self, error): self.__listener.onDownloadError(error) - def extractMetaData(self, link): - if 'hotstar' in link: + def extractMetaData(self, link, qual, name): + if "hotstar" in link or "sonyliv" in link: self.opts['geo_bypass_country'] = 'IN' with YoutubeDL(self.opts) as ydl: try: result = ydl.extract_info(link, download=False) - name = ydl.prepare_filename(result) + if name == "": + name = ydl.prepare_filename(result) + else: + name = name + # noobway hack for changing extension after converting to mp3 + if qual == "audio": + name = name.replace(".mp4", ".mp3").replace(".webm", ".mp3") except DownloadError as e: self.onDownloadError(str(e)) return @@ -111,7 +123,7 @@ def extractMetaData(self, link): if 'entries' in result: video = result['entries'][0] for v in result['entries']: - if v.get('filesize'): + if v and v.get('filesize'): self.size += float(v['filesize']) # For playlists, ydl.prepare-filename returns the following format: -.NA self.name = name.split(f"-{result['id']}")[0] @@ -138,11 +150,19 @@ def __download(self, link): LOGGER.info("Download Cancelled by User!") self.onDownloadError("Download Cancelled by User!") - def add_download(self, link, path): + def add_download(self, link, path, qual, name): + pattern = '^.*(youtu\.be\/|youtube.com\/)(playlist?)' + if re.match(pattern, link): + self.opts['ignoreerrors'] = True self.__onDownloadStart() - self.extractMetaData(link) + self.extractMetaData(link, qual, name) LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" + if qual == "audio": + self.opts['format'] = 'bestaudio/best' + self.opts['postprocessors'] = [{'key': 'FFmpegExtractAudio','preferredcodec': 'mp3','preferredquality': '192',}] + else: + self.opts['format'] = qual if not self.is_playlist: self.opts['outtmpl'] = f"{path}/{self.name}" else: diff --git a/bot/helper/mirror_utils/status_utils/extract_status.py b/bot/helper/mirror_utils/status_utils/extract_status.py index f7f5e59d6..41b6ff591 100644 --- a/bot/helper/mirror_utils/status_utils/extract_status.py +++ b/bot/helper/mirror_utils/status_utils/extract_status.py @@ -8,7 +8,7 @@ def __init__(self, name, path, size): self.__path = path self.__size = size - # The progress of Tar function cannot be tracked. So we just return dummy values. + # The progress of extract function cannot be tracked. So we just return dummy values. # If this is possible in future,we should implement it def progress(self): diff --git a/bot/helper/mirror_utils/status_utils/gdownload_status.py b/bot/helper/mirror_utils/status_utils/gdownload_status.py new file mode 100644 index 000000000..5793ce289 --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/gdownload_status.py @@ -0,0 +1,63 @@ +import bot + +from .status import Status +from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time +from bot import DOWNLOAD_DIR + + +class DownloadStatus(Status): + def __init__(self, obj, size, listener, gid): + self.dobj = obj + self.__dsize = size + self.uid = listener.uid + self.message = listener.message + self.__dgid = gid + + def path(self): + return f"{DOWNLOAD_DIR}{self.uid}" + + def processed_bytes(self): + return self.dobj.downloaded_bytes + + def size_raw(self): + return self.__dsize + + def size(self): + return get_readable_file_size(self.__dsize) + + def status(self): + return MirrorStatus.STATUS_DOWNLOADING + + def name(self): + return self.dobj.name + + def gid(self) -> str: + return self.__dgid + + def progress_raw(self): + try: + return self.dobj.downloaded_bytes / self.__dsize * 100 + except ZeroDivisionError: + return 0 + + def progress(self): + return f'{round(self.progress_raw(), 2)}%' + + def speed_raw(self): + """ + :return: Download speed in Bytes/Seconds + """ + return self.dobj.dspeed() + + def speed(self): + return f'{get_readable_file_size(self.speed_raw())}/s' + + def eta(self): + try: + seconds = (self.__dsize - self.dobj.downloaded_bytes) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' + + def download(self): + return self.dobj \ No newline at end of file diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_status.py similarity index 50% rename from bot/helper/mirror_utils/status_utils/mega_download_status.py rename to bot/helper/mirror_utils/status_utils/mega_status.py index bb5cf7311..832c67e4c 100644 --- a/bot/helper/mirror_utils/status_utils/mega_download_status.py +++ b/bot/helper/mirror_utils/status_utils/mega_status.py @@ -1,62 +1,56 @@ -from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time from bot import DOWNLOAD_DIR +from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status class MegaDownloadStatus(Status): - - def __init__(self, obj,listener): - self.uid = obj.uid - self.listener = listener + def __init__(self, obj, listener): self.obj = obj + self.uid = listener.uid + self.message = listener.message - def name(self) -> str: - return self.obj.name - - def progress_raw(self): - try: - return round(self.processed_bytes() / self.obj.size * 100,2) - except ZeroDivisionError: - return 0.0 - - def progress(self): - """Progress of download in percentage""" - return f"{self.progress_raw()}%" - + def gid(self): + return self.obj.gid - def status(self) -> str: - return MirrorStatus.STATUS_DOWNLOADING + def path(self): + return f"{DOWNLOAD_DIR}{self.uid}" def processed_bytes(self): return self.obj.downloaded_bytes - def eta(self): - try: - seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() - return f'{get_readable_time(seconds)}' - except ZeroDivisionError: - return '-' - def size_raw(self): return self.obj.size - def size(self) -> str: + def size(self): return get_readable_file_size(self.size_raw()) - def downloaded(self) -> str: - return get_readable_file_size(self.obj.downloadedBytes) + def status(self): + return MirrorStatus.STATUS_DOWNLOADING - def speed_raw(self): - return self.obj.speed + def name(self): + return self.obj.name - def speed(self) -> str: - return f'{get_readable_file_size(self.speed_raw())}/s' + def progress_raw(self): + return self.obj.progress - def gid(self) -> str: - return self.obj.gid + def progress(self): + return f'{round(self.progress_raw(), 2)}%' - def path(self) -> str: - return f"{DOWNLOAD_DIR}{self.uid}" + def speed_raw(self): + """ + :return: Download speed in Bytes/Seconds + """ + return self.obj.download_speed + + def speed(self): + return f'{get_readable_file_size(self.speed_raw())}/s' + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' def download(self): return self.obj \ No newline at end of file diff --git a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py index a29013d79..3950baddd 100644 --- a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py +++ b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py @@ -1,7 +1,7 @@ from bot import DOWNLOAD_DIR from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status - +from bot.helper.ext_utils.fs_utils import get_path_size class YoutubeDLDownloadStatus(Status): def __init__(self, obj, listener): @@ -16,7 +16,10 @@ def path(self): return f"{DOWNLOAD_DIR}{self.uid}" def processed_bytes(self): - return self.obj.downloaded_bytes + if self.obj.downloaded_bytes != 0: + return self.obj.downloaded_bytes + else: + return get_path_size(f"{DOWNLOAD_DIR}{self.uid}") def size_raw(self): return self.obj.size @@ -49,7 +52,7 @@ def eta(self): try: seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() return f'{get_readable_time(seconds)}' - except ZeroDivisionError: + except: return '-' def download(self): diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 5f30a5c58..d7a0018c3 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -1,4 +1,5 @@ import os +import io import pickle import urllib.parse as urlparse from urllib.parse import parse_qs @@ -6,23 +7,31 @@ import re import json import requests +import logging +from random import randrange from google.auth.transport.requests import Request from google.oauth2 import service_account from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from googleapiclient.http import MediaFileUpload +from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from tenacity import * +from telegram import InlineKeyboardMarkup +from bot.helper.telegram_helper import button_build +from telegraph import Telegraph + from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ - USE_SERVICE_ACCOUNTS + USE_SERVICE_ACCOUNTS, download_dict, telegraph_token, BUTTON_THREE_NAME, BUTTON_THREE_URL, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, SHORTENER, SHORTENER_API from bot.helper.ext_utils.bot_utils import * -from bot.helper.ext_utils.fs_utils import get_mime_type +from bot.helper.ext_utils.fs_utils import get_mime_type, get_path_size LOGGER = logging.getLogger(__name__) logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) -SERVICE_ACCOUNT_INDEX = 0 +if USE_SERVICE_ACCOUNTS: + SERVICE_ACCOUNT_INDEX = randrange(len(os.listdir("accounts"))) +TELEGRAPHLIMIT = 95 class GoogleDriveHelper: @@ -39,18 +48,30 @@ def __init__(self, name=None, listener=None): self.__service = self.authorize() self.__listener = listener self._file_uploaded_bytes = 0 - self.__uploaded_bytes = 0 + self._file_downloaded_bytes = 0 + self.uploaded_bytes = 0 + self.downloaded_bytes = 0 + self.UPDATE_INTERVAL = 5 self.start_time = 0 + self.total_time = 0 + self.dtotal_time = 0 + self._should_update = True + self.is_uploading = True self.is_cancelled = False + self.status = None + self.dstatus = None + self.updater = None self.name = name - self.transferred_size = 0 + self.update_interval = 3 + self.telegraph_content = [] + self.path = [] + self.total_bytes = 0 + self.total_files = 0 + self.total_folders = 0 def cancel(self): self.is_cancelled = True - - @property - def uploaded_bytes(self): - return self.__uploaded_bytes + self.is_uploading = False def speed(self): """ @@ -58,15 +79,22 @@ def speed(self): :return: Upload speed in bytes/second """ try: - return self.uploaded_bytes / (time.time() - self.start_time) + return self.uploaded_bytes / self.total_time + except ZeroDivisionError: + return 0 + + def dspeed(self): + try: + return self.downloaded_bytes / self.dtotal_time except ZeroDivisionError: return 0 + @staticmethod def getIdFromUrl(link: str): if "folders" in link or "file" in link: regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" - res = re.search(regex, link) + res = re.search(regex,link) if res is None: raise IndexError("GDrive ID not found.") return res.group(5) @@ -75,13 +103,21 @@ def getIdFromUrl(link: str): @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def _on_upload_progress(self): + if self.status is not None: + chunk_size = self.status.total_size * self.status.progress() - self._file_uploaded_bytes + self._file_uploaded_bytes = self.status.total_size * self.status.progress() + LOGGER.debug(f'Uploading {self.name}, chunk size: {get_readable_file_size(chunk_size)}') + self.uploaded_bytes += chunk_size + self.total_time += self.update_interval + def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): media_body = MediaFileUpload(path, mimetype=mime_type, resumable=False) file_metadata = { 'name': file_name, - 'description': 'mirror', + 'description': 'mirrored', 'mimeType': mime_type, } if parent_id is not None: @@ -144,27 +180,21 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): drive_file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata, media_body=media_body) response = None - last_uploaded = 0 while response is None: if self.is_cancelled: return None try: - status, response = drive_file.next_chunk() + self.status, response = drive_file.next_chunk() except HttpError as err: - status = None if err.resp.get('content-type', '').startswith('application/json'): reason = json.loads(err.content).get('error').get('errors')[0].get('reason') if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.upload_file(file_path, file_name, mime_type, parent_id) + return self.upload_file(file_path, file_name, mime_type, parent_id) else: raise err - if status is not None: - chunk_size = status.total_size * status.progress() - last_uploaded - last_uploaded = status.total_size * status.progress() - self.__uploaded_bytes += chunk_size self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: @@ -174,19 +204,42 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url + def deletefile(self, link: str): + try: + file_id = self.getIdFromUrl(link) + except (KeyError,IndexError): + msg = "Google drive ID could not be found in the provided link" + return msg + msg = '' + try: + res = self.__service.files().delete(fileId=file_id, supportsTeamDrives=IS_TEAM_DRIVE).execute() + msg = "Successfully deleted" + except HttpError as err: + LOGGER.error(str(err)) + if "File not found" in str(err): + msg = "No such file exist" + else: + msg = "Something went wrong check log" + finally: + return msg + def upload(self, file_name: str): + if USE_SERVICE_ACCOUNTS: + self.service_account_count = len(os.listdir("accounts")) self.__listener.onUploadStarted() file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" file_path = f"{file_dir}/{file_name}" + size = get_readable_file_size(get_path_size(file_path)) LOGGER.info("Uploading File: " + file_path) self.start_time = time.time() + self.updater = setInterval(self.update_interval, self._on_upload_progress) if os.path.isfile(file_path): try: mime_type = get_mime_type(file_path) link = self.upload_file(file_path, file_name, mime_type, parent_id) if link is None: raise Exception('Upload has been manually cancelled') - LOGGER.info("Uploaded To G-Drive: " + file_path) + LOGGER.info("Uploaded To G-Drive:- " + file_path) except Exception as e: if isinstance(e, RetryError): LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") @@ -196,6 +249,8 @@ def upload(self, file_name: str): LOGGER.error(err) self.__listener.onUploadError(str(err)) return + finally: + self.updater.cancel() else: try: dir_id = self.create_directory(os.path.basename(os.path.abspath(file_name)), parent_id) @@ -213,8 +268,10 @@ def upload(self, file_name: str): LOGGER.error(err) self.__listener.onUploadError(str(err)) return + finally: + self.updater.cancel() LOGGER.info(download_dict) - self.__listener.onUploadComplete(link) + self.__listener.onUploadComplete(link, size) LOGGER.info("Deleting downloaded file/folder..") return link @@ -226,7 +283,7 @@ def copyFile(self, file_id, dest_id): } try: - res = self.__service.files().copy(supportsAllDrives=True, fileId=file_id, body=body).execute() + res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() return res except HttpError as err: if err.resp.get('content-type', '').startswith('application/json'): @@ -235,73 +292,28 @@ def copyFile(self, file_id, dest_id): if USE_SERVICE_ACCOUNTS: self.switchServiceAccount() LOGGER.info(f"Got: {reason}, Trying Again.") - self.copyFile(file_id, dest_id) + return self.copyFile(file_id,dest_id) else: raise err - def clone(self, link): - self.transferred_size = 0 - try: - file_id = self.getIdFromUrl(link) - except (KeyError, IndexError): - msg = "Google drive ID could not be found in the provided link" - return msg - msg = "" - LOGGER.info(f"File ID: {file_id}") - try: - meta = self.__service.files().get(supportsAllDrives=True, fileId=file_id, + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), + retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def getFileMetadata(self,file_id): + return self.__service.files().get(supportsAllDrives=True, fileId=file_id, fields="name,id,mimeType,size").execute() - except Exception as e: - return f"{str(e).replace('>', '').replace('<', '')}" - if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: - dir_id = self.create_directory(meta.get('name'), parent_id) - try: - result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) - except Exception as e: - if isinstance(e, RetryError): - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - err = e.last_attempt.exception() - else: - err = str(e).replace('>', '').replace('<', '') - LOGGER.error(err) - return err - msg += f'{meta.get("name")}' \ - f' ({get_readable_file_size(self.transferred_size)})' - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') - msg += f' | Index URL' - else: - try: - file = self.copyFile(meta.get('id'), parent_id) - except Exception as e: - if isinstance(e, RetryError): - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - err = e.last_attempt.exception() - else: - err = str(e).replace('>', '').replace('<', '') - LOGGER.error(err) - return err - msg += f'{file.get("name")}' - try: - msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') - msg += f' | Index URL' - except TypeError: - pass - return msg - def cloneFolder(self, name, local_path, folder_id, parent_id): + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), + retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def getFilesByFolderId(self,folder_id): page_token = None q = f"'{folder_id}' in parents" files = [] - LOGGER.info(f"Syncing: {local_path}") - new_id = None while True: response = self.__service.files().list(supportsTeamDrives=True, includeTeamDriveItems=True, q=q, spaces='drive', + pageSize=200, fields='nextPageToken, files(id, name, mimeType,size)', pageToken=page_token).execute() for file in response.get('files', []): @@ -309,6 +321,87 @@ def cloneFolder(self, name, local_path, folder_id, parent_id): page_token = response.get('nextPageToken', None) if page_token is None: break + return files + + def clone(self, link): + self.transferred_size = 0 + self.total_files = 0 + self.total_folders = 0 + try: + file_id = self.getIdFromUrl(link) + except (KeyError,IndexError): + msg = "Google drive ID could not be found in the provided link" + return msg, "" + msg = "" + LOGGER.info(f"File ID: {file_id}") + try: + meta = self.getFileMetadata(file_id) + if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: + dir_id = self.create_directory(meta.get('name'), parent_id) + result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) + msg += f'Name:- {meta.get("name")}\nSize:- {get_readable_file_size(self.transferred_size)}' + durl = self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL.format(dir_id) + buttons = button_build.ButtonMaker() + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={durl}&format=text').text + buttons.buildbutton("Drive Link", surl) + else: + buttons.buildbutton("Drive Link", durl) + if INDEX_URL is not None: + url_path = requests.utils.quote(f'{meta.get("name")}') + url = f'{INDEX_URL}/{url_path}/' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + buttons.buildbutton("Index Link", siurl) + else: + buttons.buildbutton("Index Link", url) + if BUTTON_THREE_NAME is not None and BUTTON_THREE_URL is not None: + buttons.buildbutton(f"{BUTTON_THREE_NAME}", f"{BUTTON_THREE_URL}") + if BUTTON_FOUR_NAME is not None and BUTTON_FOUR_URL is not None: + buttons.buildbutton(f"{BUTTON_FOUR_NAME}", f"{BUTTON_FOUR_URL}") + if BUTTON_FIVE_NAME is not None and BUTTON_FIVE_URL is not None: + buttons.buildbutton(f"{BUTTON_FIVE_NAME}", f"{BUTTON_FIVE_URL}") + else: + file = self.copyFile(meta.get('id'), parent_id) + msg += f'Name:- {file.get("name")}' + durl = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(file.get("id")) + buttons = button_build.ButtonMaker() + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={durl}&format=text').text + buttons.buildbutton("Drive Link", surl) + else: + buttons.buildbutton("Drive Link", durl) + try: + msg += f'\nSize:- {get_readable_file_size(int(meta.get("size")))}' + except TypeError: + pass + if INDEX_URL is not None: + url_path = requests.utils.quote(f'{file.get("name")}') + url = f'{INDEX_URL}/{url_path}' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + buttons.buildbutton("Index Link", siurl) + else: + buttons.buildbutton("Index Link", url) + if BUTTON_THREE_NAME is not None and BUTTON_THREE_URL is not None: + buttons.buildbutton(f"{BUTTON_THREE_NAME}", f"{BUTTON_THREE_URL}") + if BUTTON_FOUR_NAME is not None and BUTTON_FOUR_URL is not None: + buttons.buildbutton(f"{BUTTON_FOUR_NAME}", f"{BUTTON_FOUR_URL}") + if BUTTON_FIVE_NAME is not None and BUTTON_FIVE_URL is not None: + buttons.buildbutton(f"{BUTTON_FIVE_NAME}", f"{BUTTON_FIVE_URL}") + except Exception as err: + if isinstance(err, RetryError): + LOGGER.info(f"Total Attempts:- {err.last_attempt.attempt_number}") + err = err.last_attempt.exception() + err = str(err).replace('>', '').replace('<', '') + LOGGER.error(err) + return err, "" + return msg, InlineKeyboardMarkup(buttons.build_menu(2)) + + def cloneFolder(self, name, local_path, folder_id, parent_id): + LOGGER.info(f"Syncing: {local_path}") + files = self.getFilesByFolderId(folder_id) + new_id = None if len(files) == 0: return parent_id for file in files: @@ -395,30 +488,318 @@ def authorize(self): scopes=self.__OAUTH_SCOPE) return build('drive', 'v3', credentials=credentials, cache_discovery=False) + def edit_telegraph(self): + nxt_page = 1 + prev_page = 0 + for content in self.telegraph_content : + if nxt_page == 1 : + content += f'Next' + nxt_page += 1 + else : + if prev_page <= self.num_of_path: + content += f'Prev' + prev_page += 1 + if nxt_page < self.num_of_path: + content += f' | Next' + nxt_page += 1 + Telegraph(access_token=telegraph_token).edit_page(path = self.path[prev_page], + title = 'MirrorX Search', + author_name='MirrorX', + author_url='https://github.com/iamLiquidX', + html_content=content) + return + + def escapes(self, str): + chars = ['\\', "'", '"', r'\a', r'\b', r'\f', r'\n', r'\r', r'\t'] + for char in chars: + str = str.replace(char, '\\'+char) + return str + def drive_list(self, fileName): msg = "" + fileName = self.escapes(str(fileName)) # Create Search Query for API request. query = f"'{parent_id}' in parents and (name contains '{fileName}')" response = self.__service.files().list(supportsTeamDrives=True, includeTeamDriveItems=True, q=query, spaces='drive', - pageSize=20, + pageSize=200, fields='files(id, name, mimeType, size)', orderBy='modifiedTime desc').execute() - for file in response.get('files', []): - if file.get( - 'mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. - msg += f"⁍ {file.get('name')}" \ - f" (folder)" - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') - msg += f' | Index URL' + + content_count = 0 + if response["files"]: + msg += f'

Results : {fileName}



' + + for file in response.get('files', []): + if file.get('mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. + furl = f"https://drive.google.com/drive/folders/{file.get('id')}" + msg += f"⁍{file.get('name')}
(folder)
" + if SHORTENER is not None and SHORTENER_API is not None: + sfurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={furl}&format=text').text + msg += f"Drive Link" + else: + msg += f"Drive Link" + if INDEX_URL is not None: + url_path = requests.utils.quote(f'{file.get("name")}') + url = f'{INDEX_URL}/{url_path}/' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + msg += f' | Index Link' + else: + msg += f' | Index Link' + else: + furl = f"https://drive.google.com/uc?id={file.get('id')}&export=download" + msg += f"⁍{file.get('name')}
({get_readable_file_size(int(file.get('size')))})
" + if SHORTENER is not None and SHORTENER_API is not None: + sfurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={furl}&format=text').text + msg += f"Drive Link" + else: + msg += f"Drive Link" + if INDEX_URL is not None: + url_path = requests.utils.quote(f'{file.get("name")}') + url = f'{INDEX_URL}/{url_path}' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={url}&format=text').text + msg += f' | Index Link' + else: + msg += f' | Index Link' + msg += '

' + content_count += 1 + if content_count == TELEGRAPHLIMIT : + self.telegraph_content.append(msg) + msg = "" + content_count = 0 + + if msg != '': + self.telegraph_content.append(msg) + + if len(self.telegraph_content) == 0: + return "No Result Found :(", None + + for content in self.telegraph_content : + self.path.append(Telegraph(access_token=telegraph_token).create_page( + title = 'MirrorX Search', + author_name='MirrorX', + author_url='https://github.com/iamLiquidX', + html_content=content + )['path']) + + self.num_of_path = len(self.path) + if self.num_of_path > 1: + self.edit_telegraph() + + msg = f"Search Results For {fileName} 👇" + buttons = button_build.ButtonMaker() + buttons.buildbutton("HERE", f"https://telegra.ph/{self.path[0]}") + + return msg, InlineKeyboardMarkup(buttons.build_menu(1)) + + else : + return '', '' + + + + def count(self, link): + self.total_bytes = 0 + self.total_files = 0 + self.total_folders = 0 + try: + file_id = self.getIdFromUrl(link) + except (KeyError,IndexError): + msg = "Google drive ID could not be found in the provided link" + return msg + msg = "" + LOGGER.info(f"File ID: {file_id}") + try: + drive_file = self.__service.files().get(fileId=file_id, fields="id, name, mimeType, size", + supportsTeamDrives=True).execute() + name = drive_file['name'] + LOGGER.info(f"Counting: {name}") + if drive_file['mimeType'] == self.__G_DRIVE_DIR_MIME_TYPE: + self.gDrive_directory(**drive_file) + msg += f'Name:- {name}' + msg += f'\nSize:- {get_readable_file_size(self.total_bytes)}' + msg += f"\nType:- Folder" + msg += f"\nSubFolders:- {self.total_folders}" + msg += f"\nFiles:- {self.total_files}" else: - msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') - msg += f' | Index URL' - msg += '\n' + msg += f'Name:- {name}' + try: + typee = drive_file['mimeType'] + except: + typee = 'File' + try: + self.total_files += 1 + self.gDrive_file(**drive_file) + msg += f'\nSize:- {get_readable_file_size(self.total_bytes)}' + msg += f"\nType:- {typee}" + msg += f"\nFiles:- {self.total_files}" + except TypeError: + pass + except Exception as err: + if isinstance(err, RetryError): + LOGGER.info(f"Total Attempts: {err.last_attempt.attempt_number}") + err = err.last_attempt.exception() + err = str(err).replace('>', '').replace('<', '') + LOGGER.error(err) + return err return msg + + def gDrive_file(self, **kwargs): + try: + size = int(kwargs['size']) + except: + size = 0 + self.total_bytes += size + + def gDrive_directory(self, **kwargs) -> None: + files = self.getFilesByFolderId(kwargs['id']) + if len(files) == 0: + return + for file_ in files: + if file_['mimeType'] == self.__G_DRIVE_DIR_MIME_TYPE: + self.total_folders += 1 + self.gDrive_directory(**file_) + else: + self.total_files += 1 + self.gDrive_file(**file_) + + def clonehelper(self, link): + try: + file_id = self.getIdFromUrl(link) + except (KeyError,IndexError): + msg = "Google drive ID could not be found in the provided link" + return msg, "", "" + LOGGER.info(f"File ID: {file_id}") + try: + drive_file = self.__service.files().get(fileId=file_id, fields="id, name, mimeType, size", + supportsTeamDrives=True).execute() + name = drive_file['name'] + LOGGER.info(f"Checking: {name}") + if drive_file['mimeType'] == self.__G_DRIVE_DIR_MIME_TYPE: + self.gDrive_directory(**drive_file) + else: + try: + self.total_files += 1 + self.gDrive_file(**drive_file) + except TypeError: + pass + clonesize = self.total_bytes + except Exception as err: + if isinstance(err, RetryError): + LOGGER.info(f"Total Attempts: {err.last_attempt.attempt_number}") + err = err.last_attempt.exception() + err = str(err).replace('>', '').replace('<', '') + LOGGER.error(err) + if "File not found" in str(err): + msg = "File not found." + else: + msg = f"Error.\n{err}" + return msg, "", "" + return "", clonesize, name + + def download(self, link): + self.is_downloading = True + file_id = self.getIdFromUrl(link) + if USE_SERVICE_ACCOUNTS: + self.service_account_count = len(os.listdir("accounts")) + self.start_time = time.time() + self.updater = setInterval(self.update_interval, self._on_download_progress) + try: + meta = self.getFileMetadata(file_id) + path = f"{DOWNLOAD_DIR}{self.__listener.uid}/" + if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: + self.download_folder(file_id, path, meta.get('name')) + else: + os.makedirs(path) + self.download_file(file_id, path, meta.get('name'), meta.get('mimeType')) + except Exception as err: + if isinstance(err, RetryError): + LOGGER.info(f"Total Attempts: {err.last_attempt.attempt_number}") + err = err.last_attempt.exception() + err = str(err).replace('>', '').replace('<', '') + LOGGER.error(err) + self.is_cancelled = True + self.__listener.onDownloadError(err) + return + finally: + self.updater.cancel() + if self.is_cancelled: + return + self.__listener.onDownloadComplete() + + def download_folder(self, folder_id, path, folder_name): + if not os.path.exists(path + folder_name): + os.makedirs(path + folder_name) + path += folder_name + '/' + result = [] + page_token = None + while True: + files = self.__service.files().list( + supportsTeamDrives=True, + includeTeamDriveItems=True, + q=f"'{folder_id}' in parents", + fields='nextPageToken, files(id, name, mimeType, size, shortcutDetails)', + pageToken=page_token, + pageSize=1000).execute() + result.extend(files['files']) + page_token = files.get("nextPageToken") + if not page_token: + break + + result = sorted(result, key=lambda k: k['name']) + for item in result: + file_id = item['id'] + filename = item['name'] + mime_type = item['mimeType'] + shortcut_details = item.get('shortcutDetails', None) + if shortcut_details != None: + file_id = shortcut_details['targetId'] + mime_type = shortcut_details['targetMimeType'] + if mime_type == 'application/vnd.google-apps.folder': + self.download_folder(file_id, path, filename) + elif not os.path.isfile(path + filename): + self.download_file(file_id, path, filename, mime_type) + if self.is_cancelled: + break + return + + def download_file(self, file_id, path, filename, mime_type): + request = self.__service.files().get_media(fileId=file_id) + fh = io.FileIO('{}{}'.format(path, filename), 'wb') + downloader = MediaIoBaseDownload(fh, request, chunksize = 65 * 1024 * 1024) + done = False + while done is False: + if self.is_cancelled: + fh.close() + break + return + try: + self.dstatus, done = downloader.next_chunk() + except HttpError as err: + if err.resp.get('content-type', '').startswith('application/json'): + reason = json.loads(err.content).get('error').get('errors')[0].get('reason') + if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': + if USE_SERVICE_ACCOUNTS: + if not self.switchServiceAccount(): + raise err + LOGGER.info(f"Got: {reason}, Trying Again...") + return self.download_file(file_id, path, filename, mime_type) + else: + raise err + else: + raise err + self._file_downloaded_bytes = 0 + + def _on_download_progress(self): + if self.dstatus is not None: + chunk_size = self.dstatus.total_size * self.dstatus.progress() - self._file_downloaded_bytes + self._file_downloaded_bytes = self.dstatus.total_size * self.dstatus.progress() + self.downloaded_bytes += chunk_size + self.dtotal_time += self.update_interval + + def cancel_download(self): + self.is_cancelled = True + self.__listener.onDownloadError('Download stopped by user!') diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py index bd1af0c25..9589629d2 100644 --- a/bot/helper/telegram_helper/bot_commands.py +++ b/bot/helper/telegram_helper/bot_commands.py @@ -1,22 +1,34 @@ +import os + +def getCommand(name: str, command: str): + try: + if len(os.environ[name]) == 0: + raise KeyError + return os.environ[name] + except KeyError: + return command class _BotCommands: def __init__(self): - self.StartCommand = 'start' - self.MirrorCommand = 'mirror' - self.UnzipMirrorCommand = 'unzipmirror' - self.TarMirrorCommand = 'tarmirror' - self.CancelMirror = 'cancel' - self.CancelAllCommand = 'cancelall' - self.ListCommand = 'list' - self.StatusCommand = 'status' - self.AuthorizeCommand = 'authorize' - self.UnAuthorizeCommand = 'unauthorize' - self.PingCommand = 'ping' - self.RestartCommand = 'restart' - self.StatsCommand = 'stats' - self.HelpCommand = 'help' - self.LogCommand = 'log' - self.CloneCommand = "clone" - self.WatchCommand = 'watch' - self.TarWatchCommand = 'tarwatch' + self.StartCommand = getCommand('START_COMMAND', 'start') + self.MirrorCommand = getCommand('MIRROR_COMMAND', 'mirror') + self.UnzipMirrorCommand = getCommand('UNZIP_COMMAND', 'unzipmirror') + self.TarMirrorCommand = getCommand('TAR_COMMAND', 'tarmirror') + self.CancelMirror = getCommand('CANCEL_COMMAND', 'cancel') + self.CancelAllCommand = getCommand('CANCELALL_COMMAND', 'cancelall') + self.ListCommand = getCommand('LIST_COMMAND', 'list') + self.SpeedCommand = getCommand('SPEED_COMMAND', 'speedtest') + self.CountCommand = getCommand('COUNT_COMMAND', 'count') + self.StatusCommand = getCommand('STATUS_COMMAND', 'status') + self.AuthorizeCommand = getCommand('AUTH_COMMAND', 'authorize') + self.UnAuthorizeCommand = getCommand('UNAUTH_COMMAND', 'unauthorize') + self.PingCommand = getCommand('PING_COMMAND', 'ping') + self.RestartCommand = getCommand('RESTART_COMMAND', 'restart') + self.StatsCommand = getCommand('STATS_COMMAND', 'stats') + self.HelpCommand = getCommand('HELP_COMMAND', 'help') + self.LogCommand = getCommand('LOG_COMMAND', 'log') + self.CloneCommand = getCommand('CLONE_COMMAND', 'clone') + self.WatchCommand = getCommand('WATCH_COMMAND', 'watch') + self.TarWatchCommand = getCommand('TARWATCH_COMMAND', 'tarwatch') + self.deleteCommand = getCommand('DELETE_COMMAND', 'del') BotCommands = _BotCommands() diff --git a/bot/helper/telegram_helper/button_build.py b/bot/helper/telegram_helper/button_build.py new file mode 100644 index 000000000..1b85b520c --- /dev/null +++ b/bot/helper/telegram_helper/button_build.py @@ -0,0 +1,16 @@ +from telegram import InlineKeyboardButton + +class ButtonMaker: + def __init__(self): + self.button = [] + + def buildbutton(self, key, link): + self.button.append(InlineKeyboardButton(text = key, url = link)) + + def build_menu(self, n_cols, footer_buttons=None, header_buttons=None): + menu = [self.button[i:i + n_cols] for i in range(0, len(self.button), n_cols)] + if header_buttons: + menu.insert(0, header_buttons) + if footer_buttons: + menu.append(footer_buttons) + return menu \ No newline at end of file diff --git a/bot/helper/telegram_helper/filters.py b/bot/helper/telegram_helper/filters.py index 3cd3c9363..b40b0217f 100644 --- a/bot/helper/telegram_helper/filters.py +++ b/bot/helper/telegram_helper/filters.py @@ -1,29 +1,29 @@ -from telegram.ext import BaseFilter +from telegram.ext import MessageFilter from telegram import Message from bot import AUTHORIZED_CHATS, OWNER_ID, download_dict, download_dict_lock class CustomFilters: - class _OwnerFilter(BaseFilter): + class _OwnerFilter(MessageFilter): def filter(self, message): return bool(message.from_user.id == OWNER_ID) owner_filter = _OwnerFilter() - class _AuthorizedUserFilter(BaseFilter): + class _AuthorizedUserFilter(MessageFilter): def filter(self, message): id = message.from_user.id return bool(id in AUTHORIZED_CHATS or id == OWNER_ID) authorized_user = _AuthorizedUserFilter() - class _AuthorizedChat(BaseFilter): + class _AuthorizedChat(MessageFilter): def filter(self, message): return bool(message.chat.id in AUTHORIZED_CHATS) authorized_chat = _AuthorizedChat() - class _MirrorOwner(BaseFilter): + class _MirrorOwner(MessageFilter): def filter(self, message: Message): user_id = message.from_user.id if user_id == OWNER_ID: diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index 9d3eb02c5..cc1a82663 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -1,12 +1,12 @@ +from telegram import InlineKeyboardMarkup from telegram.message import Message from telegram.update import Update import time +import psutil from bot import AUTO_DELETE_MESSAGE_DURATION, LOGGER, bot, \ - status_reply_dict, status_reply_dict_lock -from bot.helper.ext_utils.bot_utils import get_readable_message + status_reply_dict, status_reply_dict_lock, download_dict, download_dict_lock +from bot.helper.ext_utils.bot_utils import get_readable_message, get_readable_file_size, MirrorStatus from telegram.error import TimedOut, BadRequest -from bot import bot - def sendMessage(text: str, bot, update: Update): try: @@ -17,10 +17,19 @@ def sendMessage(text: str, bot, update: Update): LOGGER.error(str(e)) -def editMessage(text: str, message: Message): +def sendMarkup(text: str, bot, update: Update, reply_markup: InlineKeyboardMarkup): + try: + return bot.send_message(update.message.chat_id, + reply_to_message_id=update.message.message_id, + text=text, reply_markup=reply_markup, parse_mode='HTMl') + except Exception as e: + LOGGER.error(str(e)) + + +def editMessage(text: str, message: Message, reply_markup=None): try: bot.edit_message_text(text=text, message_id=message.message_id, - chat_id=message.chat.id, + chat_id=message.chat.id,reply_markup=reply_markup, parse_mode='HTMl') except Exception as e: LOGGER.error(str(e)) @@ -64,9 +73,32 @@ def delete_all_messages(): def update_all_messages(): msg = get_readable_message() + msg += f"CPU:- {psutil.cpu_percent()}%" \ + f" DISK:- {psutil.disk_usage('/').percent}%" \ + f" RAM:- {psutil.virtual_memory().percent}%" + with download_dict_lock: + dlspeed_bytes = 0 + uldl_bytes = 0 + for download in list(download_dict.values()): + speedy = download.speed() + if download.status() == MirrorStatus.STATUS_DOWNLOADING: + if 'KiB/s' in speedy: + dlspeed_bytes += float(speedy.split('K')[0]) * 1024 + elif 'MiB/s' in speedy: + dlspeed_bytes += float(speedy.split('M')[0]) * 1048576 + if download.status() == MirrorStatus.STATUS_UPLOADING: + if 'KB/s' in speedy: + uldl_bytes += float(speedy.split('K')[0]) * 1024 + elif 'MB/s' in speedy: + uldl_bytes += float(speedy.split('M')[0]) * 1048576 + dlspeed = get_readable_file_size(dlspeed_bytes) + ulspeed = get_readable_file_size(uldl_bytes) + msg += f"\nDL:{dlspeed}ps | UL:{ulspeed}ps \n" with status_reply_dict_lock: for chat_id in list(status_reply_dict.keys()): if status_reply_dict[chat_id] and msg != status_reply_dict[chat_id].text: + if len(msg) == 0: + msg = "Starting DL" try: editMessage(msg, status_reply_dict[chat_id]) except Exception as e: @@ -76,6 +108,27 @@ def update_all_messages(): def sendStatusMessage(msg, bot): progress = get_readable_message() + progress += f"CPU: {psutil.cpu_percent()}%" \ + f" DISK: {psutil.disk_usage('/').percent}%" \ + f" RAM: {psutil.virtual_memory().percent}%" + with download_dict_lock: + dlspeed_bytes = 0 + uldl_bytes = 0 + for download in list(download_dict.values()): + speedy = download.speed() + if download.status() == MirrorStatus.STATUS_DOWNLOADING: + if 'KiB/s' in speedy: + dlspeed_bytes += float(speedy.split('K')[0]) * 1024 + elif 'MiB/s' in speedy: + dlspeed_bytes += float(speedy.split('M')[0]) * 1048576 + if download.status() == MirrorStatus.STATUS_UPLOADING: + if 'KB/s' in speedy: + uldl_bytes += float(speedy.split('K')[0]) * 1024 + elif 'MB/s' in speedy: + uldl_bytes += float(speedy.split('M')[0]) * 1048576 + dlspeed = get_readable_file_size(dlspeed_bytes) + ulspeed = get_readable_file_size(uldl_bytes) + progress += f"\nDL:{dlspeed}ps | UL:{ulspeed}ps \n" with status_reply_dict_lock: if msg.message.chat.id in list(status_reply_dict.keys()): try: @@ -85,6 +138,7 @@ def sendStatusMessage(msg, bot): except Exception as e: LOGGER.error(str(e)) del status_reply_dict[msg.message.chat.id] - pass + if len(progress) == 0: + progress = "Starting DL" message = sendMessage(progress, bot, msg) status_reply_dict[msg.message.chat.id] = message diff --git a/bot/modules/authorize.py b/bot/modules/authorize.py index fedb4d1ac..92679f429 100644 --- a/bot/modules/authorize.py +++ b/bot/modules/authorize.py @@ -1,5 +1,4 @@ from bot.helper.telegram_helper.message_utils import sendMessage -from telegram.ext import run_async from bot import AUTHORIZED_CHATS, dispatcher from telegram.ext import CommandHandler from bot.helper.telegram_helper.filters import CustomFilters @@ -8,7 +7,6 @@ from bot.helper.telegram_helper.bot_commands import BotCommands -@run_async def authorize(update,context): reply_message = update.message.reply_to_message msg = '' @@ -34,7 +32,6 @@ def authorize(update,context): sendMessage(msg, context.bot, update) -@run_async def unauthorize(update,context): reply_message = update.message.reply_to_message if reply_message is None: @@ -61,9 +58,9 @@ def unauthorize(update,context): authorize_handler = CommandHandler(command=BotCommands.AuthorizeCommand, callback=authorize, - filters=CustomFilters.owner_filter & Filters.group) + filters=CustomFilters.owner_filter & Filters.group, run_async=True) unauthorize_handler = CommandHandler(command=BotCommands.UnAuthorizeCommand, callback=unauthorize, - filters=CustomFilters.owner_filter & Filters.group) + filters=CustomFilters.owner_filter & Filters.group, run_async=True) dispatcher.add_handler(authorize_handler) dispatcher.add_handler(unauthorize_handler) diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index a4d2d7415..8d0d3a671 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -1,4 +1,4 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from bot import download_dict, dispatcher, download_dict_lock, DOWNLOAD_DIR from bot.helper.ext_utils.fs_utils import clean_download @@ -10,7 +10,6 @@ from bot.helper.ext_utils.bot_utils import getDownloadByGid, MirrorStatus -@run_async def cancel_mirror(update, context): args = update.message.text.split(" ", maxsplit=1) mirror_message = None @@ -51,7 +50,6 @@ def cancel_mirror(update, context): clean_download(f'{DOWNLOAD_DIR}{mirror_message.message_id}/') -@run_async def cancel_all(update, context): with download_dict_lock: count = 0 @@ -65,8 +63,8 @@ def cancel_all(update, context): cancel_mirror_handler = CommandHandler(BotCommands.CancelMirror, cancel_mirror, - filters=(CustomFilters.authorized_chat | CustomFilters.authorized_user) & CustomFilters.mirror_owner_filter) + filters=(CustomFilters.authorized_chat | CustomFilters.authorized_user) & CustomFilters.mirror_owner_filter, run_async=True) cancel_all_handler = CommandHandler(BotCommands.CancelAllCommand, cancel_all, - filters=CustomFilters.owner_filter) + filters=CustomFilters.owner_filter, run_async=True) dispatcher.add_handler(cancel_all_handler) dispatcher.add_handler(cancel_mirror_handler) diff --git a/bot/modules/clone.py b/bot/modules/clone.py index 407e3532a..1ecd8c1f9 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -1,23 +1,32 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper from bot.helper.telegram_helper.message_utils import * from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.bot_commands import BotCommands +from bot.helper.ext_utils.bot_utils import new_thread from bot import dispatcher -@run_async def cloneNode(update,context): args = update.message.text.split(" ",maxsplit=1) if len(args) > 1: link = args[1] msg = sendMessage(f"Cloning: {link}",context.bot,update) gd = GoogleDriveHelper() - result = gd.clone(link) + result, button = gd.clone(link) deleteMessage(context.bot,msg) - sendMessage(result,context.bot,update) + if button == "": + sendMessage(result,context.bot,update) + else: + if update.message.from_user.username: + uname = f'@{update.message.from_user.username}' + else: + uname = f'{update.message.from_user.first_name}' + if uname is not None: + cc = f'\n\nReq. By: {uname}' + sendMarkup(result + cc, context.bot, update, button) else: - sendMessage("Provide G-Drive Shareable Link to Clone.",bot,update) + sendMessage("Provide G-Drive Shareable Link to Clone.",context.bot,update) -clone_handler = CommandHandler(BotCommands.CloneCommand,cloneNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) -dispatcher.add_handler(clone_handler) \ No newline at end of file +clone_handler = CommandHandler(BotCommands.CloneCommand,cloneNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) +dispatcher.add_handler(clone_handler) diff --git a/bot/modules/count.py b/bot/modules/count.py new file mode 100644 index 000000000..fe64e3535 --- /dev/null +++ b/bot/modules/count.py @@ -0,0 +1,29 @@ +import bot + +from telegram.ext import CommandHandler +from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper +from bot.helper.telegram_helper.message_utils import deleteMessage, sendMessage +from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.telegram_helper.bot_commands import BotCommands +from bot import dispatcher, AUTHORIZED_CHATS + +def countNode(update,context): + args = update.message.text.split(" ",maxsplit=1) + if len(args) > 1: + link = args[1] + msg = sendMessage(f"Counting: {link}",context.bot,update) + gd = GoogleDriveHelper() + result = gd.count(link) + deleteMessage(context.bot,msg) + if update.message.from_user.username: + uname = f'@{update.message.from_user.username}' + else: + uname = f'{update.message.from_user.first_name}' + if uname is not None: + cc = f'\n\nReq. By: {uname}' + sendMessage(result + cc, context.bot, update) + else: + sendMessage("Provide G-Drive Shareable Link to Count.",context.bot,update) + +count_handler = CommandHandler(BotCommands.CountCommand,countNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) +dispatcher.add_handler(count_handler) \ No newline at end of file diff --git a/bot/modules/delete.py b/bot/modules/delete.py new file mode 100644 index 000000000..eb4d263a1 --- /dev/null +++ b/bot/modules/delete.py @@ -0,0 +1,29 @@ +from telegram.ext import CommandHandler +import threading +from telegram import Update +from bot import dispatcher, LOGGER +from bot.helper.telegram_helper.message_utils import auto_delete_message, sendMessage +from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.telegram_helper.bot_commands import BotCommands +from bot.helper.mirror_utils.upload_utils import gdriveTools + +def deletefile(update, context): + msg_args = update.message.text.split(None, 1) + msg = '' + try: + link = msg_args[1] + LOGGER.info(msg_args[1]) + except IndexError: + msg = 'send a link along with command' + + if msg == '' : + drive = gdriveTools.GoogleDriveHelper() + msg = drive.deletefile(link) + LOGGER.info(f"this is msg : {msg}") + reply_message = sendMessage(msg, context.bot, update) + + threading.Thread(target=auto_delete_message, args=(context.bot, update.message, reply_message)).start() + +delete_handler = CommandHandler(command=BotCommands.deleteCommand, callback=deletefile, + filters=CustomFilters.owner_filter, run_async=True) +dispatcher.add_handler(delete_handler) diff --git a/bot/modules/list.py b/bot/modules/list.py index 24a641c0a..9ed9bc7aa 100644 --- a/bot/modules/list.py +++ b/bot/modules/list.py @@ -1,25 +1,28 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper from bot import LOGGER, dispatcher -from bot.helper.telegram_helper.message_utils import auto_delete_message, sendMessage +from bot.helper.telegram_helper.message_utils import sendMessage, sendMarkup, editMessage from bot.helper.telegram_helper.filters import CustomFilters -import threading from bot.helper.telegram_helper.bot_commands import BotCommands -@run_async def list_drive(update,context): - message = update.message.text - search = message.split(' ',maxsplit=1)[1] - LOGGER.info(f"Searching: {search}") - gdrive = GoogleDriveHelper(None) - msg = gdrive.drive_list(search) - if msg: - reply_message = sendMessage(msg, context.bot, update) - else: - reply_message = sendMessage('No result found', context.bot, update) + try: + search = update.message.text.split(' ',maxsplit=1)[1] + LOGGER.info(f"Searching: {search}") + reply = sendMessage('Searching..... Please wait!', context.bot, update) + gdrive = GoogleDriveHelper(None) + msg, button = gdrive.drive_list(search) - threading.Thread(target=auto_delete_message, args=(context.bot, update.message, reply_message)).start() + if button: + editMessage(msg, reply, button) + else: + editMessage('No result found', reply, button) + except IndexError: + sendMessage('send a search key along with command', context.bot, update) + except AttributeError: + pass -list_handler = CommandHandler(BotCommands.ListCommand, list_drive,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + +list_handler = CommandHandler(BotCommands.ListCommand, list_drive,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) dispatcher.add_handler(list_handler) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 4cf23574d..fe7ff363d 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -1,36 +1,46 @@ import requests -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler +from telegram import InlineKeyboardMarkup -from bot import Interval, INDEX_URL -from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock +from bot import Interval, INDEX_URL, BUTTON_THREE_NAME, BUTTON_THREE_URL, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, BLOCK_MEGA_LINKS, MEGA_KEY +from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock, SHORTENER, SHORTENER_API from bot.helper.ext_utils import fs_utils, bot_utils from bot.helper.ext_utils.bot_utils import setInterval -from bot.helper.ext_utils.exceptions import DirectDownloadLinkException +from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper -from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper +from bot.helper.mirror_utils.download_utils.mega_download import MegaDownloader from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners from bot.helper.mirror_utils.status_utils.extract_status import ExtractStatus from bot.helper.mirror_utils.status_utils.tar_status import TarStatus from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus +from bot.helper.mirror_utils.status_utils.gdownload_status import DownloadStatus from bot.helper.mirror_utils.upload_utils import gdriveTools from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * +from bot.helper.telegram_helper import button_build +import urllib import pathlib import os +import subprocess +import threading +import re +import random +import string ariaDlManager = AriaDownloadHelper() ariaDlManager.start_listener() class MirrorListener(listeners.MirrorListeners): - def __init__(self, bot, update, isTar=False,tag=None, extract=False): + def __init__(self, bot, update, pswd, isTar=False, tag=None, extract=False): super().__init__(bot, update) self.isTar = isTar self.tag = tag self.extract = extract + self.pswd = pswd def onDownloadStarted(self): pass @@ -53,7 +63,9 @@ def onDownloadComplete(self): download = download_dict[self.uid] name = download.name() size = download.size_raw() - m_path = f'{DOWNLOAD_DIR}{self.uid}/{download.name()}' + if name is None: # when pyrogram's media.file_name is of NoneType + name = os.listdir(f'{DOWNLOAD_DIR}{self.uid}')[0] + m_path = f'{DOWNLOAD_DIR}{self.uid}/{name}' if self.isTar: download.is_archiving = True try: @@ -66,35 +78,40 @@ def onDownloadComplete(self): return elif self.extract: download.is_extracting = True - - path = fs_utils.get_base_name(m_path) - if path != "unsupported": + try: + path = fs_utils.get_base_name(m_path) LOGGER.info( - f"Extracting : {download_dict[self.uid].name()} " + f"Extracting : {name} " ) - download_dict[self.uid] = ExtractStatus(name, m_path, size) - os.system(f"extract '{m_path}'") - if not os.path.exists(path): - self.onUploadError("Cannot extract file, check integrity of the file") - return + with download_dict_lock: + download_dict[self.uid] = ExtractStatus(name, m_path, size) + pswd = self.pswd + if pswd is not None: + archive_result = subprocess.run(["pextract", m_path, pswd]) + else: + archive_result = subprocess.run(["extract", m_path]) + if archive_result.returncode == 0: + threading.Thread(target=os.remove, args=(m_path,)).start() + LOGGER.info(f"Deleting archive : {m_path}") + else: + LOGGER.warning('Unable to extract archive! Uploading anyway') + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' LOGGER.info( f'got path : {path}' ) - try: - os.remove(m_path) - LOGGER.info(f"Deleting archive : {m_path}") - except Exception as e: - LOGGER.error(str(e)) - else: + + except NotSupportedExtractionArchive: LOGGER.info("Not any valid archive, uploading file as it is.") - path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' else: - path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' + path = f'{DOWNLOAD_DIR}{self.uid}/{name}' up_name = pathlib.PurePath(path).name + if up_name == "None": + up_name = "".join(os.listdir(f'{DOWNLOAD_DIR}{self.uid}/')) + up_path = f'{DOWNLOAD_DIR}{self.uid}/{up_name}' LOGGER.info(f"Upload Name : {up_name}") drive = gdriveTools.GoogleDriveHelper(up_name, self) - if size == 0: - size = fs_utils.get_path_size(m_path) + size = fs_utils.get_path_size(up_path) upload_status = UploadStatus(drive, size, self) with download_dict_lock: download_dict[self.uid] = upload_status @@ -133,28 +150,45 @@ def onUploadStarted(self): def onUploadProgress(self): pass - def onUploadComplete(self, link: str): + def onUploadComplete(self, link: str, size): with download_dict_lock: - msg = f'{download_dict[self.uid].name()} ({download_dict[self.uid].size()})' + msg = f'Name:- {download_dict[self.uid].name()}\nSize:- {size}' + buttons = button_build.ButtonMaker() + if SHORTENER is not None and SHORTENER_API is not None: + surl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={link}&format=text').text + buttons.buildbutton("Drive Link", surl) + else: + buttons.buildbutton("Drive Link", link) LOGGER.info(f'Done Uploading {download_dict[self.uid].name()}') if INDEX_URL is not None: - share_url = requests.utils.requote_uri(f'{INDEX_URL}/{download_dict[self.uid].name()}') + url_path = requests.utils.quote(f'{download_dict[self.uid].name()}') + share_url = f'{INDEX_URL}/{url_path}' if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): share_url += '/' - msg += f'\n\n Shareable link: here' + if SHORTENER is not None and SHORTENER_API is not None: + siurl = requests.get(f'https://{SHORTENER}/api?api={SHORTENER_API}&url={share_url}&format=text').text + buttons.buildbutton("Index Link", siurl) + else: + buttons.buildbutton("Index Link", share_url) + if BUTTON_THREE_NAME is not None and BUTTON_THREE_URL is not None: + buttons.buildbutton(f"{BUTTON_THREE_NAME}", f"{BUTTON_THREE_URL}") + if BUTTON_FOUR_NAME is not None and BUTTON_FOUR_URL is not None: + buttons.buildbutton(f"{BUTTON_FOUR_NAME}", f"{BUTTON_FOUR_URL}") + if BUTTON_FIVE_NAME is not None and BUTTON_FIVE_URL is not None: + buttons.buildbutton(f"{BUTTON_FIVE_NAME}", f"{BUTTON_FIVE_URL}") if self.message.from_user.username: uname = f"@{self.message.from_user.username}" else: uname = f'{self.message.from_user.first_name}' if uname is not None: - msg += f'\n\ncc : {uname}' + msg += f'\n\nReq. By:- {uname}' try: fs_utils.clean_download(download_dict[self.uid].path()) except FileNotFoundError: pass del download_dict[self.uid] count = len(download_dict) - sendMessage(msg, self.bot, self.update) + sendMarkup(msg, self.bot, self.update, InlineKeyboardMarkup(buttons.build_menu(2))) if count == 0: self.clean() else: @@ -175,12 +209,38 @@ def onUploadError(self, error): else: update_all_messages() + def _mirror(bot, update, isTar=False, extract=False): - message_args = update.message.text.split(' ') + mesg = update.message.text.split('\n') + message_args = mesg[0].split(' ') + name_args = mesg[0].split('|') try: link = message_args[1] + print(link) + if link.startswith("|") or link.startswith("pswd: "): + link = '' except IndexError: link = '' + try: + name = name_args[1] + name = name.strip() + if name.startswith("pswd: "): + name = '' + except IndexError: + name = '' + try: + ussr = urllib.parse.quote(mesg[1], safe='') + pssw = urllib.parse.quote(mesg[2], safe='') + except: + ussr = '' + pssw = '' + if ussr != '' and pssw != '': + link = link.split("://", maxsplit=1) + link = f'{link[0]}://{ussr}:{pssw}@{link[1]}' + pswd = re.search('(?<=pswd: )(.*)', update.message.text) + if pswd is not None: + pswd = pswd.groups() + pswd = " ".join(pswd) LOGGER.info(link) link = link.strip() reply_to = update.message.reply_to_message @@ -193,12 +253,12 @@ def _mirror(bot, update, isTar=False, extract=False): file = i break - if len(link) == 0: + if not bot_utils.is_url(link) and not bot_utils.is_magnet(link) or len(link) == 0: if file is not None: if file.mime_type != "application/x-bittorrent": - listener = MirrorListener(bot, update, isTar, tag) + listener = MirrorListener(bot, update, pswd, isTar, tag, extract) tg_downloader = TelegramDownloadHelper(listener) - tg_downloader.add_download(reply_to, f'{DOWNLOAD_DIR}{listener.uid}/') + tg_downloader.add_download(reply_to, f'{DOWNLOAD_DIR}{listener.uid}/', name) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) @@ -215,38 +275,58 @@ def _mirror(bot, update, isTar=False, extract=False): link = direct_link_generator(link) except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') - listener = MirrorListener(bot, update, isTar, tag, extract) - if bot_utils.is_mega_link(link): - mega_dl = MegaDownloadHelper() - mega_dl.add_download(link,f'{DOWNLOAD_DIR}/{listener.uid}/',listener) + listener = MirrorListener(bot, update, pswd, isTar, tag, extract) + if bot_utils.is_gdrive_link(link): + if not isTar and not extract: + sendMessage(f"Use /{BotCommands.CloneCommand} To Copy File/Folder", bot, update) + return + res, size, name = gdriveTools.GoogleDriveHelper().clonehelper(link) + if res != "": + sendMessage(res, bot, update) + return + LOGGER.info(f"Download Name : {name}") + drive = gdriveTools.GoogleDriveHelper(name, listener) + gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=12)) + download_status = DownloadStatus(drive, size, listener, gid) + with download_dict_lock: + download_dict[listener.uid] = download_status + if len(Interval) == 0: + Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) + sendStatusMessage(update, bot) + drive.download(link) + + elif bot_utils.is_mega_link(link) and MEGA_KEY is not None: + if BLOCK_MEGA_LINKS: + sendMessage("Mega Links Are Blocked.", bot, update) + else: + mega_dl = MegaDownloader(listener) + mega_dl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}/') + sendStatusMessage(update, bot) else: - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener) - sendStatusMessage(update, bot) + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}/', listener, name) + sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) -@run_async def mirror(update, context): _mirror(context.bot, update) -@run_async def tar_mirror(update, context): _mirror(context.bot, update, True) -@run_async def unzip_mirror(update, context): - _mirror(context.bot,update, extract=True) + _mirror(context.bot, update, extract=True) mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) tar_mirror_handler = CommandHandler(BotCommands.TarMirrorCommand, tar_mirror, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) unzip_mirror_handler = CommandHandler(BotCommands.UnzipMirrorCommand, unzip_mirror, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) dispatcher.add_handler(mirror_handler) dispatcher.add_handler(tar_mirror_handler) dispatcher.add_handler(unzip_mirror_handler) diff --git a/bot/modules/mirror_status.py b/bot/modules/mirror_status.py index e4ae32494..482d17ad6 100644 --- a/bot/modules/mirror_status.py +++ b/bot/modules/mirror_status.py @@ -1,4 +1,4 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from bot import dispatcher, status_reply_dict, DOWNLOAD_STATUS_UPDATE_INTERVAL, status_reply_dict_lock from bot.helper.telegram_helper.message_utils import * from time import sleep @@ -8,7 +8,6 @@ from bot.helper.telegram_helper.bot_commands import BotCommands import threading -@run_async def mirror_status(update,context): message = get_readable_message() if len(message) == 0: @@ -26,5 +25,5 @@ def mirror_status(update,context): mirror_status_handler = CommandHandler(BotCommands.StatusCommand, mirror_status, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) dispatcher.add_handler(mirror_status_handler) diff --git a/bot/modules/speedtest.py b/bot/modules/speedtest.py new file mode 100644 index 000000000..d597dc460 --- /dev/null +++ b/bot/modules/speedtest.py @@ -0,0 +1,48 @@ +import speedtest +import bot + +from bot.helper.telegram_helper.filters import CustomFilters +from bot import dispatcher, AUTHORIZED_CHATS +from bot.helper.telegram_helper.bot_commands import BotCommands +from telegram import Update, InlineKeyboardButton, InlineKeyboardMarkup, ParseMode +from telegram.ext import CallbackContext, Filters, CommandHandler + + + +def speedtst(update, context): + message = update.effective_message + ed_msg = message.reply_text("Running Speed Test . . . 💨") + test = speedtest.Speedtest() + test.get_best_server() + test.download() + test.upload() + test.results.share() + result = test.results.dict() + context.bot.editMessageText( + "🔻 Download Speed : " + f"{speed_convert(result['download'])}\n" + "🔺 Upload Speed : " + f"{speed_convert(result['upload'])}\n" + "📶 Ping : " + f"{result['ping']}\n" + "🏬 ISP : " + f"{result['client']['isp']}", + update.effective_chat.id, + ed_msg.message_id, + ) + +def speed_convert(size): + """Hi human, you can't read bytes?""" + power = 2 ** 10 + zero = 0 + units = {0: "", 1: "Kb/s", 2: "Mb/s", 3: "Gb/s", 4: "Tb/s"} + while size > power: + size /= power + zero += 1 + return f"{round(size, 2)} {units[zero]}" + + +SPEED_HANDLER = CommandHandler(BotCommands.SpeedCommand, speedtst, + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) + +dispatcher.add_handler(SPEED_HANDLER) diff --git a/bot/modules/watch.py b/bot/modules/watch.py index 42bdb1d70..d4139d606 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -1,4 +1,4 @@ -from telegram.ext import CommandHandler, run_async +from telegram.ext import CommandHandler from telegram import Bot, Update from bot import Interval, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, dispatcher, LOGGER from bot.helper.ext_utils.bot_utils import setInterval @@ -10,40 +10,61 @@ import threading -def _watch(bot: Bot, update: Update, args: list, isTar=False): +def _watch(bot: Bot, update, isTar=False): + mssg = update.message.text + message_args = mssg.split(' ') + name_args = mssg.split('|') try: - link = args[0] + link = message_args[1] except IndexError: - sendMessage(f'/{BotCommands.WatchCommand} [yt_dl supported link] to mirror with youtube_dl', bot, update) + msg = f"/{BotCommands.WatchCommand} [yt_dl supported link] [quality] |[CustomName] to mirror with youtube_dl.\n\n" + msg += "Note :- Quality and custom name are optional\n\nExample of quality :- audio, 144, 240, 360, 480, 720, 1080, 2160." + msg += "\n\nIf you want to use custom filename, plz enter it after |" + msg += f"\n\nExample :-\n/{BotCommands.WatchCommand} https://youtu.be/ocX2FN1nguA 720 |My video bro\n\n" + msg += "This file will be downloaded in 720p quality and it's name will be My video bro" + sendMessage(msg, bot, update) return + try: + if "|" in mssg: + mssg = mssg.split("|") + qual = mssg[0].split(" ")[2] + if qual == "": + raise IndexError + else: + qual = message_args[2] + if qual != "audio": + qual = f'bestvideo[height<={qual}]+bestaudio/best[height<={qual}]' + except IndexError: + qual = "bestvideo+bestaudio/best" + try: + name = name_args[1] + except IndexError: + name = "" reply_to = update.message.reply_to_message if reply_to is not None: tag = reply_to.from_user.username else: tag = None - - listener = MirrorListener(bot, update, isTar, tag) + pswd = "" + listener = MirrorListener(bot, update, pswd, isTar, tag) ydl = YoutubeDLHelper(listener) - threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}')).start() + threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}', qual, name)).start() sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) -@run_async def watchTar(update, context): - _watch(context.bot, update, context.args, True) + _watch(context.bot, update, True) def watch(update, context): - _watch(context.bot, update, context.args) + _watch(context.bot, update) mirror_handler = CommandHandler(BotCommands.WatchCommand, watch, - pass_args=True, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) tar_mirror_handler = CommandHandler(BotCommands.TarWatchCommand, watchTar, - pass_args=True, - filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True) dispatcher.add_handler(mirror_handler) dispatcher.add_handler(tar_mirror_handler) diff --git a/captain-definition b/captain-definition deleted file mode 100644 index 0e14f8239..000000000 --- a/captain-definition +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schemaVersion": 2, - "dockerfilePath": "./Dockerfile" -} diff --git a/config_sample.env b/config_sample.env index 453c888c9..50057266c 100644 --- a/config_sample.env +++ b/config_sample.env @@ -1,17 +1,66 @@ #Remove this line before deploying _____REMOVE_THIS_LINE_____=True -# ENTER BOT TOKEN (Get your BOT_TOKEN by talking to @botfather) -BOT_TOKEN = "" -GDRIVE_FOLDER_ID = "" +## Telegram +BOT_TOKEN = "" # ENTER BOT TOKEN (Get your BOT_TOKEN by talking to @botfather) OWNER_ID = -DOWNLOAD_DIR = "/home/username/mirror-bot/downloads" DOWNLOAD_STATUS_UPDATE_INTERVAL = 5 AUTO_DELETE_MESSAGE_DURATION = 20 -IS_TEAM_DRIVE = "" -INDEX_URL = "" -USER_SESSION_STRING = "" TELEGRAM_API = TELEGRAM_HASH = "" +AUTHORIZED_CHATS = "" #Separated by space +MAX_CONCURRENT_DOWNLOADS = 4 + +## Drive +GDRIVE_FOLDER_ID = "" +IS_TEAM_DRIVE = "" +INDEX_URL = "" +DOWNLOAD_DIR = "/app/downloads" # Dont change this if you are using Docker compose USE_SERVICE_ACCOUNTS = "" -MEGA_API_KEY = "" + +## Optional config +UPTOBOX_TOKEN = "" + +MEGA_KEY = "" # Get this from https://mega.nz/sdk +MEGA_USERNAME = "" +MEGA_PASSWORD = "" +BLOCK_MEGA_LINKS = "" + +STOP_DUPLICATE_MIRROR = "" + +SHORTENER = "" +SHORTENER_API = "" + +# Add more buttons (two buttons are already added of file link and index link, you can add extra buttons too, these are optional) +# If you don't know what are below entries, simply leave them, Don't fill anything in them. +BUTTON_THREE_NAME = "" +BUTTON_THREE_URL = "" +BUTTON_FOUR_NAME = "" +BUTTON_FOUR_URL = "" +BUTTON_FIVE_NAME = "" +BUTTON_FIVE_URL = "" + + +# Commands Customization (Completely Optional) +# If you want to change +START_COMMAND = "" +MIRROR_COMMAND = "" +UNZIP_COMMAND = "" +TAR_COMMAND = "" +CANCEL_COMMAND = "" +CANCELALL_COMMAND = "" +LIST_COMMAND = "" +SPEED_COMMAND = "" +COUNT_COMMAND = "" +STATUS_COMMAND = "" +AUTH_COMMAND = "" +UNAUTH_COMMAND = "" +PING_COMMAND = "" +RESTART_COMMAND = "" +STATS_COMMAND = "" +HELP_COMMAND = "" +LOG_COMMAND = "" +CLONE_COMMAND = "" +WATCH_COMMAND = "" +TARWATCH_COMMAND = "" +DELETE_COMMAND = "" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..27b72f5ee --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.9" + +services: + mirrorx: + image: ghcr.io/iamliquidx/mirrorx:latest + container_name: mirrorx + env_file: + # Should be filled up and should be kept in the same directory + - config.env + volumes: + # Make a folder in the host OS and keep SA(accounts folder) or token there + # syntax "host OS path":"where it should be mounted inside container" + - /home/someuser/mirrorx/accounts:/app/accounts + # for Service Accounts ex: /home/someuser/mirrorx/accounts:/app/accounts + # for token and credential.json + # - /home/someuser/mirrorx/token.pickle:/app/token.pickle + # - /home/someuser/mirrorx/credential.json:/app/credentials.json + # Incase you want to add a netrc file to youtube-dl + # - /home/someuser/mirrorx/netrc:/root/.netrc + # Dont edit the Container Directory (i.e Right Side one) as the bot runs in the same directory + + restart: unless-stopped + # Restart policy, if container exits due to some error it will restart again \ No newline at end of file diff --git a/extract b/extract deleted file mode 100755 index db113eb88..000000000 --- a/extract +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -if [ $# -lt 1 ];then - echo "Usage: `basename $0` FILES" - exit 1 -fi - -extract () { - arg="$1" - cd $(dirname "$arg") - case "$arg" in - *.tar.bz2) - tar xjf "$arg" --one-top-level ;; - *.tar.gz) - tar xzf "$arg" --one-top-level ;; - *.bz2) bunzip2 "$arg" ;; - *.gz) gunzip "$arg" ;; - *.tar) tar xf "$arg" --one-top-level ;; - *.tbz2) tar xjf "$arg" --one-top-level ;; - *.tgz) tar xzf "$arg" --one-top-level ;; - *.zip) - a_dir=`expr "$arg" : '\(.*\).zip'` - unzip "$arg" -d "$a_dir" ;; - *.Z) uncompress "$arg" ;; - *.rar) - a_dir=`expr "$arg" : '\(.*\).rar'` ; - mkdir "$a_dir" ; - rar x "$arg" "$a_dir" ;; # 'rar' must to be installed - *) echo "'$arg' cannot be extracted via extract()" ;; - esac - cd - -} - -extract "$1" diff --git a/generate_string_session.py b/generate_string_session.py deleted file mode 100644 index fd6d57c55..000000000 --- a/generate_string_session.py +++ /dev/null @@ -1,6 +0,0 @@ -from pyrogram import Client - -API_KEY = int(input("Enter API KEY: ")) -API_HASH = input("Enter API HASH: ") -with Client(':memory:', api_id=API_KEY, api_hash=API_HASH) as app: - print(app.export_session_string()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1699c9b6d..cbc8ef944 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,24 @@ -requests -python-telegram-bot==12.6.1 -google-api-python-client>=1.7.11,<1.7.20 -google-auth-httplib2>=0.0.3,<0.1.0 -google-auth-oauthlib>=0.4.1,<0.10.0 -aria2p>=0.9.0,<0.15.0 -python-dotenv>=0.10 -tenacity>=6.0.0 -python-magic -beautifulsoup4>=4.8.2,<4.8.10 -Pyrogram>=0.16.0,<0.16.10 -TgCrypto>=1.1.1,<1.1.10 -youtube-dl +requests==2.26.0 +appdirs==1.4.4 +aria2p==0.10.4 +progress==1.6 +psutil==5.8.0 +python-telegram-bot==13.7 +google-api-python-client==2.11.0 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.4.5 +js2py==0.71 +python-dotenv==0.19.0 +tenacity==8.0.1 +python-magic==0.4.24 +beautifulsoup4==4.9.3 +Pyrogram +TgCrypto +yt-dlp +lxml==4.6.3 +telegraph==1.4.1 +speedtest-cli==2.1.3 +messages==0.5.0 +pybase64==1.1.4 +lk21==1.6.0 +megasdkrestclient \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..41cc779ac --- /dev/null +++ b/setup.py @@ -0,0 +1,67 @@ +from setuptools import setup, find_packages +import pathlib + +CWD = pathlib.Path(__file__).parent + +README = (CWD / "README.md").read_text() + +setup( + name='MirrorX', + version='6.0.13', + packages=find_packages(), + long_description=README, + long_description_content_type="text/markdown", + url='https://github.com/iamliquidx/mirrorx', + license='GPL3.0', + author='', + author_email='', + include_package_data=True, + description='Telegram Mirror Bot', + platforms="any", + install_requires=[ + "requests==2.26.0", + "appdirs==1.4.4", + "aria2p==0.10.4", + "progress==1.6", + "psutil==5.8.0", + "python-telegram-bot==13.7", + "google-api-python-client==2.11.0", + "google-auth-httplib2==0.1.0", + "google-auth-oauthlib==0.4.5", + "js2py==0.71", + "python-dotenv==0.19.0", + "tenacity==8.0.1", + "python-magic==0.4.24", + "beautifulsoup4==4.9.3", + "Pyrogram", + "TgCrypto", + "yt-dlp", + "lxml==4.6.3", + "telegraph==1.4.1", + "speedtest-cli==2.1.3", + "messages==0.5.0", + "pybase64==1.1.4", + "lk21==1.6.0", + "megasdkrestclient" + ], + classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: GNU General Public License (GPL)", + "Operating System :: POSIX :: Linux", + "Development Status :: 5 - Production/Stable" + ], + python_requires=">=3.7", + entry_points={ + "console_scripts":[ + "MirrorXBot = bot.__main__:main" + ] + + }, + package_data={ + "": ["data/*.dat", "data/aria.conf"], + }, + scripts=['bin/extract', 'bin/pextract', 'bin/MirrorX'], +) diff --git a/start.sh b/start.sh deleted file mode 100755 index 25b805fc6..000000000 --- a/start.sh +++ /dev/null @@ -1 +0,0 @@ -./aria.sh; python3 -m bot \ No newline at end of file diff --git a/add_to_team_drive.py b/utils/add_to_team_drive.py similarity index 96% rename from add_to_team_drive.py rename to utils/add_to_team_drive.py index 222cbe1b1..f461a127d 100644 --- a/add_to_team_drive.py +++ b/utils/add_to_team_drive.py @@ -1,4 +1,3 @@ -from __future__ import print_function from google.oauth2.service_account import Credentials import googleapiclient.discovery, json, progress.bar, glob, sys, argparse, time from google_auth_oauthlib.flow import InstalledAppFlow @@ -23,7 +22,7 @@ credentials = glob.glob(args.credentials) try: - open(credentials[0], 'r') + open(credentials[0]) print('>> Found credentials.') except IndexError: print('>> No credentials found.') @@ -60,7 +59,7 @@ aa = glob.glob('%s/*.json' % acc_dir) pbar = progress.bar.Bar("Readying accounts", max=len(aa)) for i in aa: - ce = json.loads(open(i, 'r').read())['client_email'] + ce = json.loads(open(i).read())['client_email'] batch.add(drive.permissions().create(fileId=did, supportsAllDrives=True, body={ "role": "fileOrganizer", "type": "user", diff --git a/gen_sa_accounts.py b/utils/gen_sa_accounts.py similarity index 97% rename from gen_sa_accounts.py rename to utils/gen_sa_accounts.py index ff6f144b7..54c25c265 100644 --- a/gen_sa_accounts.py +++ b/utils/gen_sa_accounts.py @@ -96,7 +96,7 @@ def _enable_services(service, projects, ste): batch = service.new_batch_http_request(callback=_def_batch_resp) for i in projects: for j in ste: - batch.add(service.services().enable(name='projects/%s/services/%s' % (i, j))) + batch.add(service.services().enable(name=f'projects/{i}/services/{j}')) batch.execute() @@ -134,7 +134,7 @@ def _create_sa_keys(iam, projects, path): total_sas = _list_sas(iam, i) for j in total_sas: batch.add(iam.projects().serviceAccounts().keys().create( - name='projects/%s/serviceAccounts/%s' % (i, j['uniqueId']), + name='projects/{}/serviceAccounts/{}'.format(i, j['uniqueId']), body={ 'privateKeyType': 'TYPE_GOOGLE_CREDENTIALS_FILE', 'keyAlgorithm': 'KEY_ALG_RSA_2048' @@ -176,7 +176,7 @@ def serviceaccountfactory( download_keys=None ): selected_projects = [] - proj_id = loads(open(credentials, 'r').read())['installed']['project_id'] + proj_id = loads(open(credentials).read())['installed']['project_id'] creds = None if os.path.exists(token): with open(token, 'rb') as t: @@ -214,7 +214,7 @@ def serviceaccountfactory( if list_sas: return _list_sas(iam, list_sas) if create_projects: - print("creat projects: {}".format(create_projects)) + print(f"creat projects: {create_projects}") if create_projects > 0: current_count = len(_get_projects(cloud)) if current_count + create_projects <= max_projects: @@ -360,6 +360,6 @@ def serviceaccountfactory( if resp: print('Service accounts in %s (%d):' % (args.list_sas, len(resp))) for i in resp: - print(' %s (%s)' % (i['email'], i['uniqueId'])) + print(' {} ({})'.format(i['email'], i['uniqueId'])) else: print('No service accounts.') diff --git a/generate_drive_token.py b/utils/generate_drive_token.py similarity index 100% rename from generate_drive_token.py rename to utils/generate_drive_token.py diff --git a/vendor/cmrudl.py b/vendor/cmrudl.py deleted file mode 160000 index f7d75bcf7..000000000 --- a/vendor/cmrudl.py +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f7d75bcf7901aee7b1430fc17366b7b6af65235e