diff --git a/.gitignore b/.gitignore index 2a6dddb..940d56a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ node_modules/ -dist/behaviors.js +dist diff --git a/src/autoscroll.ts b/src/autoscroll.ts index 7a05fc6..7794067 100644 --- a/src/autoscroll.ts +++ b/src/autoscroll.ts @@ -1,25 +1,31 @@ -import { Behavior } from "./lib/behavior"; -import { sleep, waitUnit, xpathNode, isInViewport, waitUntil, behaviorLog, addLink } from "./lib/utils"; -import { type AutoFetcher } from "./autofetcher"; - +import { + sleep, + waitUnit, + xpathNode, + isInViewport, + waitUntil, + behaviorLog, + addLink, + currentlyFetching, +} from "./lib/utils"; +import type { AbstractBehavior } from "./lib/behavior"; +//import { type AutoFetcher } from "./autofetcher"; // =========================================================================== -export class AutoScroll extends Behavior { - autoFetcher: AutoFetcher; +export class AutoScroll implements AbstractBehavior { showMoreQuery: string; - state: { segments: number } = { segments: 1}; + state: { segments: number } = { segments: 1 }; lastScrollPos: number; samePosCount: number; origPath: string; lastMsg = ""; - constructor(autofetcher: AutoFetcher) { - super(); - - this.autoFetcher = autofetcher; + constructor() { + //super(); - this.showMoreQuery = "//*[contains(text(), 'show more') or contains(text(), 'Show more')]"; + this.showMoreQuery = + "//*[contains(text(), 'show more') or contains(text(), 'Show more')]"; this.lastScrollPos = -1; this.samePosCount = 0; @@ -29,20 +35,37 @@ export class AutoScroll extends Behavior { static id = "Autoscroll"; + static init() { + return { + state: {}, + }; + } + + static isMatch() { + return true; + } + + async awaitPageLoad(_: any) { + return; + } + currScrollPos() { return Math.round(self.scrollY + self.innerHeight); } canScrollMore() { const scrollElem = self.document.scrollingElement || self.document.body; - return this.currScrollPos() < Math.max(scrollElem.clientHeight, scrollElem.scrollHeight); + return ( + this.currScrollPos() < + Math.max(scrollElem.clientHeight, scrollElem.scrollHeight) + ); } debug(msg: string) { if (this.lastMsg === msg) { return; } - super.debug(msg); + void behaviorLog(msg, "debug"); this.lastMsg = msg; } @@ -51,15 +74,17 @@ export class AutoScroll extends Behavior { return !!self["getEventListeners"](obj).scroll; } catch (_) { // unknown, assume has listeners - this.debug("getEventListeners() not available"); + void behaviorLog("getEventListeners() not available", "debug"); return true; } } async shouldScroll() { - if (!this.hasScrollEL(self.window) && + if ( + !this.hasScrollEL(self.window) && !this.hasScrollEL(self.document) && - !this.hasScrollEL(self.document.body)) { + !this.hasScrollEL(self.document.body) + ) { return false; } @@ -69,10 +94,11 @@ export class AutoScroll extends Behavior { } const lastScrollHeight = self.document.scrollingElement.scrollHeight; - const numFetching = this.autoFetcher.numFetching; + const numFetching = currentlyFetching(); // scroll to almost end of page - const scrollEnd = (document.scrollingElement.scrollHeight * 0.98) - self.innerHeight; + const scrollEnd = + document.scrollingElement.scrollHeight * 0.98 - self.innerHeight; window.scrollTo({ top: scrollEnd, left: 0, behavior: "smooth" }); @@ -80,8 +106,10 @@ export class AutoScroll extends Behavior { await sleep(500); // scroll height changed, should scroll - if (lastScrollHeight !== self.document.scrollingElement.scrollHeight || - numFetching < this.autoFetcher.numFetching) { + if ( + lastScrollHeight !== self.document.scrollingElement.scrollHeight || + numFetching < currentlyFetching() + ) { window.scrollTo({ top: 0, left: 0, behavior: "auto" }); return true; } @@ -94,29 +122,42 @@ export class AutoScroll extends Behavior { return false; } - if ((self.window.scrollY + self["scrollHeight"]) / self.document.scrollingElement.scrollHeight < 0.90) { + if ( + (self.window.scrollY + self["scrollHeight"]) / + self.document.scrollingElement.scrollHeight < + 0.9 + ) { return false; } return true; } - async*[Symbol.asyncIterator]() { + async *run(ctx) { + const { getState } = ctx.Lib; + if (this.shouldScrollUp()) { - yield* this.scrollUp(); + yield* this.scrollUp(ctx); return; } if (await this.shouldScroll()) { - yield* this.scrollDown(); + yield* this.scrollDown(ctx); return; } - yield this.getState("Skipping autoscroll, page seems to not be responsive to scrolling events"); + yield getState( + ctx, + "Skipping autoscroll, page seems to not be responsive to scrolling events", + ); } - async* scrollDown() { - const scrollInc = Math.min(self.document.scrollingElement.clientHeight * 0.10, 30); + async *scrollDown(ctx) { + const { getState } = ctx.Lib; + const scrollInc = Math.min( + self.document.scrollingElement.clientHeight * 0.1, + 30, + ); const interval = 75; let elapsedWait = 0; @@ -128,9 +169,12 @@ export class AutoScroll extends Behavior { while (this.canScrollMore()) { if (document.location.pathname !== this.origPath) { - behaviorLog("Location Changed, stopping scroll: " + - `${document.location.pathname} != ${this.origPath}`, "info"); - addLink(document.location.href); + void behaviorLog( + "Location Changed, stopping scroll: " + + `${document.location.pathname} != ${this.origPath}`, + "info", + ); + void addLink(document.location.href); return; } @@ -146,14 +190,17 @@ export class AutoScroll extends Behavior { } if (showMoreElem && isInViewport(showMoreElem)) { - yield this.getState("Clicking 'Show More', awaiting more content"); + yield getState(ctx, "Clicking 'Show More', awaiting more content"); showMoreElem["click"](); await sleep(waitUnit); await Promise.race([ - waitUntil(() => self.document.scrollingElement.scrollHeight > scrollHeight, 500), - sleep(30000) + waitUntil( + () => self.document.scrollingElement.scrollHeight > scrollHeight, + 500, + ), + sleep(30000), ]); if (self.document.scrollingElement.scrollHeight === scrollHeight) { @@ -163,27 +210,31 @@ export class AutoScroll extends Behavior { showMoreElem = null; } - // eslint-disable-next-line self.scrollBy(scrollOpts as ScrollToOptions); await sleep(interval); if (this.state.segments === 1) { // only print this the first time - yield this.getState(`Scrolling down by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`); + yield getState( + ctx, + `Scrolling down by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`, + ); elapsedWait = 2.0; - } else { const waitSecs = elapsedWait / (this.state.segments - 1); // only add extra wait if actually changed height // check for scrolling, but allow for more time for content to appear the longer have already scrolled - this.debug(`Waiting up to ${waitSecs} seconds for more scroll segments`); + void behaviorLog( + `Waiting up to ${waitSecs} seconds for more scroll segments`, + "debug", + ); const startTime = Date.now(); await Promise.race([ waitUntil(() => this.canScrollMore(), interval), - sleep(waitSecs) + sleep(waitSecs), ]); elapsedWait += (Date.now() - startTime) * 2; @@ -203,8 +254,12 @@ export class AutoScroll extends Behavior { } } - async* scrollUp() { - const scrollInc = Math.min(self.document.scrollingElement.clientHeight * 0.10, 30); + async *scrollUp(ctx) { + const { getState } = ctx.Lib; + const scrollInc = Math.min( + self.document.scrollingElement.clientHeight * 0.1, + 30, + ); const interval = 75; const scrollOpts = { top: -scrollInc, left: 0, behavior: "auto" }; @@ -219,20 +274,22 @@ export class AutoScroll extends Behavior { lastScrollHeight = scrollHeight; } - // eslint-disable-next-line self.scrollBy(scrollOpts as ScrollToOptions); await sleep(interval); if (this.state.segments === 1) { // only print this the first time - yield this.getState(`Scrolling up by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`); + yield getState( + ctx, + `Scrolling up by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`, + ); } else { // only add extra wait if actually changed height // check for scrolling, but allow for more time for content to appear the longer have already scrolled await Promise.race([ waitUntil(() => self.scrollY > 0, interval), - sleep((this.state.segments - 1) * 2000) + sleep((this.state.segments - 1) * 2000), ]); } } diff --git a/src/index.ts b/src/index.ts index b1d0598..60a7662 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ import { Autoplay } from "./autoplay"; import { AutoScroll } from "./autoscroll"; import { AutoClick } from "./autoclick"; import { awaitLoad, sleep, behaviorLog, _setLogFunc, _setBehaviorManager, installBehaviors, addLink, checkToJsonOverride } from "./lib/utils"; -import { type Behavior, BehaviorRunner } from "./lib/behavior"; +import { BehaviorRunner } from "./lib/behavior"; import * as Lib from "./lib/utils"; @@ -44,7 +44,7 @@ export class BehaviorManager { autofetch: AutoFetcher; behaviors: any[]; loadedBehaviors: any; - mainBehavior: Behavior | BehaviorRunner | null; + mainBehavior: BehaviorRunner | null; mainBehaviorClass: any; inited: boolean; started: boolean; @@ -152,7 +152,7 @@ export class BehaviorManager { if (!siteMatch && opts.autoscroll) { behaviorLog("Using Autoscroll"); this.mainBehaviorClass = AutoScroll; - this.mainBehavior = new AutoScroll(this.autofetch); + this.mainBehavior = new BehaviorRunner(AutoScroll, {}); } if (this.mainBehavior) { @@ -213,7 +213,7 @@ export class BehaviorManager { this.selectMainBehavior(); if (this.mainBehavior?.awaitPageLoad) { behaviorLog("Waiting for custom page load via behavior"); - await this.mainBehavior.awaitPageLoad({Lib}); + await this.mainBehavior.awaitPageLoad(); } else { behaviorLog("No custom wait behavior"); } diff --git a/src/lib/behavior.ts b/src/lib/behavior.ts index f1946af..cc22c3a 100644 --- a/src/lib/behavior.ts +++ b/src/lib/behavior.ts @@ -23,7 +23,9 @@ export class Behavior extends BackgroundBehavior { _unpause: any; state: any; scrollOpts: { - behavior: string, block: string, inline: string + behavior: string; + block: string; + inline: string; }; constructor() { @@ -87,11 +89,9 @@ export class Behavior extends BackgroundBehavior { return { state: this.state, msg }; } - cleanup() { + cleanup() {} - } - - async awaitPageLoad(_: any) { + async awaitPageLoad() { // wait for initial page load here } @@ -100,12 +100,12 @@ export class Behavior extends BackgroundBehavior { self["__bx_behaviors"].load(this); } else { console.warn( - `Could not load ${this.name} behavior: window.__bx_behaviors is not initialized` + `Could not load ${this.name} behavior: window.__bx_behaviors is not initialized`, ); } } - async*[Symbol.asyncIterator]() { + async *[Symbol.asyncIterator]() { yield; } } @@ -113,37 +113,43 @@ export class Behavior extends BackgroundBehavior { // WIP: BehaviorRunner class allows for arbitrary behaviors outside of the // library to be run through the BehaviorManager -abstract class AbstractBehaviorInst { +export abstract class AbstractBehavior { + static id: String; + static isMatch: () => boolean; + static init: () => any; + abstract run: (ctx: any) => AsyncIterable; abstract awaitPageLoad?: (ctx: any) => Promise; } -interface StaticAbstractBehavior { - id: String; - isMatch: () => boolean; - init: () => any; -} +type StaticProps = { + [K in keyof T]: T[K]; +}; + +type StaticBehaviorProps = StaticProps; -type AbstractBehavior = - (new () => AbstractBehaviorInst) & StaticAbstractBehavior; +// Non-abstract constructor type +type ConcreteBehaviorConstructor = StaticBehaviorProps & { + new (): AbstractBehavior; +}; export class BehaviorRunner extends BackgroundBehavior { - inst: AbstractBehaviorInst; - behaviorProps: StaticAbstractBehavior; + inst: AbstractBehavior; + behaviorProps: ConcreteBehaviorConstructor; ctx: any; _running: any; paused: any; _unpause: any; get id() { - return (this.inst?.constructor as any).id; + return (this.inst.constructor as ConcreteBehaviorConstructor).id; } - constructor(behavior: AbstractBehavior, mainOpts = {}) { + constructor(behavior: ConcreteBehaviorConstructor, mainOpts = {}) { super(); this.behaviorProps = behavior; - this.inst = new behavior; + this.inst = new behavior(); if ( typeof this.inst.run !== "function" || @@ -152,9 +158,9 @@ export class BehaviorRunner extends BackgroundBehavior { throw Error("Invalid behavior: missing `async run*` instance method"); } - let {state, opts} = behavior.init(); + let { state, opts } = behavior.init(); state = state || {}; - opts = opts ? {...opts, ...mainOpts} : mainOpts; + opts = opts ? { ...opts, ...mainOpts } : mainOpts; // eslint-disable-next-line @typescript-eslint/no-explicit-any const log = async (data: any, type: string) => this.wrappedLog(data, type); @@ -169,11 +175,14 @@ export class BehaviorRunner extends BackgroundBehavior { wrappedLog(data: any, type = "info") { let logData; if (typeof data === "string" || data instanceof String) { - logData = {msg: data} + logData = { msg: data }; } else { logData = data; } - this.log({...logData, behavior: this.behaviorProps.id, siteSpecific: true}, type); + this.log( + { ...logData, behavior: this.behaviorProps.id, siteSpecific: true }, + type, + ); } start() { @@ -194,9 +203,9 @@ export class BehaviorRunner extends BackgroundBehavior { await this.paused; } } - this.debug({msg: "done!", behavior: this.behaviorProps.id}); + this.debug({ msg: "done!", behavior: this.behaviorProps.id }); } catch (e) { - this.error({msg: e.toString(), behavior: this.behaviorProps.id}); + this.error({ msg: e.toString(), behavior: this.behaviorProps.id }); } } @@ -217,9 +226,7 @@ export class BehaviorRunner extends BackgroundBehavior { } } - cleanup() { - - } + cleanup() {} async awaitPageLoad() { if (this.inst.awaitPageLoad) { @@ -232,7 +239,7 @@ export class BehaviorRunner extends BackgroundBehavior { self["__bx_behaviors"].load(this); } else { console.warn( - `Could not load ${this.name} behavior: window.__bx_behaviors is not initialized` + `Could not load ${this.name} behavior: window.__bx_behaviors is not initialized`, ); } } diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 6496a7a..e43f6e3 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,3 +1,5 @@ +import { type AutoFetcher } from "../autofetcher"; + let _logFunc = console.log; let _behaviorMgrClass = null; @@ -344,3 +346,14 @@ export function getState(ctx: any, msg: string, incrValue?: string) { return { state: ctx.state, msg }; } + +// =========================================================================== +let autofetcher : AutoFetcher | null = null; + +export function setAutoFetcher(newFetcher: AutoFetcher) { + autofetcher = newFetcher; +} + +export function currentlyFetching() { + return autofetcher ? autofetcher.numFetching : 0; +} diff --git a/src/site/index.ts b/src/site/index.ts index 93a8d99..22adcd0 100644 --- a/src/site/index.ts +++ b/src/site/index.ts @@ -1,16 +1,19 @@ import { FacebookTimelineBehavior } from "./facebook"; -import { InstagramPostsBehavior } from "./instagram"; +import { InstagramFeedBehavior, InstagramPostBehavior } from "./instagram"; import { TelegramBehavior } from "./telegram"; import { TwitterTimelineBehavior } from "./twitter"; import { TikTokVideoBehavior, TikTokProfileBehavior } from "./tiktok"; +import { YoutubeBehavior } from "./youtube"; const siteBehaviors = [ - InstagramPostsBehavior, + InstagramFeedBehavior, + InstagramPostBehavior, TwitterTimelineBehavior, FacebookTimelineBehavior, TelegramBehavior, TikTokVideoBehavior, - TikTokProfileBehavior + TikTokProfileBehavior, + YoutubeBehavior ]; export default siteBehaviors; diff --git a/src/site/instagram.ts b/src/site/instagram.ts index a601c6d..09251ad 100644 --- a/src/site/instagram.ts +++ b/src/site/instagram.ts @@ -1,3 +1,5 @@ +import { AutoScroll } from "../autoscroll"; + const subpostNextOnlyChevron = "//article[@role='presentation']//div[@role='presentation']/following-sibling::button"; const Q = { @@ -16,7 +18,7 @@ const Q = { pageLoadWaitUntil: "//main" }; -export class InstagramPostsBehavior { +export class InstagramFeedBehavior { maxCommentsTime: number; postOnlyWindow: any; @@ -261,13 +263,23 @@ export class InstagramPostsBehavior { } async awaitPageLoad(ctx: any) { - const { Lib, log } = ctx; - const { assertContentValid, waitUntilNode } = Lib; + await awaitInstagramLoad(ctx); + } +} - log("Waiting for Instagram to fully load"); +export class InstagramPostBehavior extends AutoScroll { + async awaitPageLoad(ctx: any) { + await awaitInstagramLoad(ctx); + } +} - await waitUntilNode(Q.pageLoadWaitUntil, document, null, 10000); +async function awaitInstagramLoad(ctx: any) { + const { Lib, log } = ctx; + const { assertContentValid, waitUntilNode } = Lib; - assertContentValid(() => !!document.querySelector("*[aria-label='New post']"), "not_logged_in"); - } + log("Waiting for Instagram to fully load"); + + await waitUntilNode(Q.pageLoadWaitUntil, document, null, 10000); + + assertContentValid(() => !!document.querySelector("*[aria-label='New post']"), "not_logged_in"); }