Skip to content

Fix LinkedIn first-page GraphQL variable replay #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Chkhikvadze wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix LinkedIn first-page GraphQL variable replay #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions chrome-extension/background.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@ async function getConfig() {

const SECRET_VARIABLE_KEY_PATTERN = /(cookie|csrf|authorization|password|secret|li_at|jsessionid|(?:auth|access|refresh|bearer)[_-]?token)/i;
const SECRET_VARIABLE_VALUE_PATTERN = /(li_at=|JSESSIONID=|Authorization\s*[:=]|Bearer\s+)/i;
const TIMESTAMP_VARIABLE_KEY_PATTERN = /(createdBefore|createdAfter|deliveredBefore|deliveredAfter|beforeTime|afterTime|timestamp)$/i;
const TIMESTAMP_VARIABLE_KEY_PATTERN = /(createdBefore|createdAfter|createdAt|deliveredBefore|deliveredAfter|deliveredAt|beforeTime|afterTime|timestamp)$/i;
const CURSOR_VARIABLE_KEY_PATTERN = /^(cursor|nextCursor|previousCursor|prevCursor|syncToken|pageToken|paginationToken|paginationCursor)$/i;
const COUNT_BEFORE_VARIABLE_KEY_PATTERN = /^countBefore$/i;
const COUNT_AFTER_VARIABLE_KEY_PATTERN = /^countAfter$/i;
const DEFAULT_GRAPHQL_FIRST_PAGE_COUNT = 20;
const DEFAULT_GRAPHQL_FIRST_PAGE_COUNT_AFTER = 0;
const SECRET_HEADER_NAME_PATTERN = /^(cookie|authorization|proxy-authorization)$/i;
const SAFE_LINKEDIN_REPLAY_HEADER_NAMES = new Set([
"x-li-lang",
Expand Down Expand Up @@ -197,6 +202,10 @@ function isSecretVariable({ key, rawValue }) {
return SECRET_VARIABLE_KEY_PATTERN.test(key) || SECRET_VARIABLE_VALUE_PATTERN.test(rawValue || "");
}

function isCursorVariableKey(key) {
return CURSOR_VARIABLE_KEY_PATTERN.test(String(key || ""));
}

function safeTemplateWrapper(value) {
const s = String(value || "");
if (!s || s.length > 40) return "";
Expand Down Expand Up @@ -240,9 +249,16 @@ function buildVariableTemplate(variablesRaw, kind) {
if (key === "mailboxUrn") return buildDynamicVariableEntry(key, "mailboxUrn", rawValue);
if (key === "conversationUrn") return buildDynamicVariableEntry(key, "conversationUrn", rawValue);
if (key === "count") {
return { key, source: "count", defaultValue: normalizeGraphQLScalar(rawValue) || 20 };
return { key, source: "count", defaultValue: DEFAULT_GRAPHQL_FIRST_PAGE_COUNT };
}
if (COUNT_BEFORE_VARIABLE_KEY_PATTERN.test(key)) {
return { key, source: "countBefore", defaultValue: DEFAULT_GRAPHQL_FIRST_PAGE_COUNT };
}
if (COUNT_AFTER_VARIABLE_KEY_PATTERN.test(key)) {
return { key, source: "countAfter", defaultValue: DEFAULT_GRAPHQL_FIRST_PAGE_COUNT_AFTER };
}
if (TIMESTAMP_VARIABLE_KEY_PATTERN.test(key)) return { key, source: "now" };
if (isCursorVariableKey(key)) return { key, source: "cursor", optional: true };
return { key, value: normalizeGraphQLScalar(rawValue) };
});
const requiredDynamicKey = kind === "conversations" ? "mailboxUrn" : "conversationUrn";
Expand Down Expand Up @@ -330,7 +346,8 @@ function redactOperatorText(value) {
.replace(/(JSESSIONID=)[^;\s]+/gi, "$1[redacted]")
.replace(/(csrf-token|csrf_token|csrf)[:=]\s*[^;\s,}]+/gi, "$1=[redacted]")
.replace(/(Authorization[:=]\s*Bearer\s+)[^\s]+/gi, "$1[redacted]")
.replace(/Bearer\s+[A-Za-z0-9._-]+/gi, "Bearer [redacted]");
.replace(/Bearer\s+[A-Za-z0-9._-]+/gi, "Bearer [redacted]")
.replace(/((?:nextCursor|previousCursor|prevCursor|cursor|syncToken|pageToken|paginationToken|paginationCursor)\s*[:=]\s*)[^;,\s)]+/gi, "$1[redacted]");
}

async function setStatus(status, error = null, action = null) {
Expand Down Expand Up @@ -529,8 +546,8 @@ chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
const VOYAGER_ME_URL = "https://www.linkedin.com/voyager/api/me";
const VOYAGER_BASE = "https://www.linkedin.com";
const CONTRACT_FRESHNESS_MS = 1000 * 60 * 60 * 24 * 7; // 7 days
const INGEST_CONVERSATIONS_PER_PAGE = 20; // First-MVP first page only.
const INGEST_MESSAGES_PER_THREAD = 20; // First-MVP first-page only.
const INGEST_CONVERSATIONS_PER_PAGE = DEFAULT_GRAPHQL_FIRST_PAGE_COUNT; // First-MVP first page only.
const INGEST_MESSAGES_PER_THREAD = DEFAULT_GRAPHQL_FIRST_PAGE_COUNT; // First-MVP first-page only.

function isContractFresh(contract) {
if (!contract) return false;
Expand Down Expand Up @@ -564,7 +581,10 @@ function renderGraphQLVariableValue(entry, replacements) {
if (entry.source === "mailboxUrn") return wrapDynamicGraphQLValue(entry, replacements.mailboxUrn);
if (entry.source === "conversationUrn") return wrapDynamicGraphQLValue(entry, replacements.conversationUrn);
if (entry.source === "count") return replacements.count ?? entry.defaultValue ?? entry.value;
if (entry.source === "countBefore") return replacements.countBefore ?? replacements.count ?? entry.defaultValue;
if (entry.source === "countAfter") return replacements.countAfter ?? entry.defaultValue ?? DEFAULT_GRAPHQL_FIRST_PAGE_COUNT_AFTER;
if (entry.source === "now") return Date.now();
if (entry.source === "cursor") return replacements[entry.key] ?? replacements.cursor;
return entry.value;
}

Expand All @@ -579,6 +599,7 @@ function buildGraphQLVariables(template, replacements, label) {
if (!entry || !entry.key) continue;
const value = renderGraphQLVariableValue(entry, replacements);
if (value === undefined || value === null || value === "") {
if (entry.optional) continue;
throw new Error(
`Captured ${label} messaging contract is missing value for ${entry.key}. Open LinkedIn Messaging to refresh the request contract, then retry Sync.`
);
Expand All @@ -598,6 +619,7 @@ function describeTemplateEntry(entry) {
const attrs = [];
if (entry.source) attrs.push(`source=${entry.source}`);
if (entry.rawPrefix || entry.rawSuffix) attrs.push("wrapped");
if (entry.optional) attrs.push("optional");
if (Object.prototype.hasOwnProperty.call(entry, "value")) attrs.push("static");
return attrs.length ? `${entry.key}{${attrs.join(",")}}` : entry.key;
}
Expand All @@ -612,23 +634,34 @@ function redactGraphQLVariablesForDiagnostics(variables) {
if (key === "mailboxUrn") return `${key}:[runtime-mailboxUrn]`;
if (key === "conversationUrn") return `${key}:[runtime-conversationUrn]`;
if (TIMESTAMP_VARIABLE_KEY_PATTERN.test(key)) return `${key}:[runtime-timestamp]`;
if (isCursorVariableKey(key)) return `${key}:[runtime-cursor]`;
const value = truncateDiagnostic(redactOperatorText(rawValue), 80);
return `${key}:${value || "[empty]"}`;
});
return `(${parts.join(",")})`;
}

function optionalVariablesOmittedFromRender(template, variables) {
if (!Array.isArray(template)) return [];
const renderedKeys = new Set(parseGraphQLVariables(variables).map((pair) => pair.key));
return template
.filter((entry) => entry?.optional && entry.key && !renderedKeys.has(entry.key))
.map((entry) => entry.key);
}

function buildLinkedInGraphQLError(label, status, contract, variables, responseText, replayDiagnostics = "") {
const queryId = label === "conversations" ? contract?.conversationsQueryId : contract?.messagesQueryId;
const template = label === "conversations" ? contract?.conversationsVariablesTemplate : contract?.messagesVariablesTemplate;
const variableKeys = Array.isArray(template) ? template.map((entry) => entry?.key).filter(Boolean).join(",") : "unknown";
const templateShape = Array.isArray(template)
? template.map(describeTemplateEntry).filter(Boolean).join(",")
: "unknown";
const omittedOptional = optionalVariablesOmittedFromRender(template, variables).join(",");
const omittedPart = omittedOptional ? ` omitted optional variables=${omittedOptional};` : "";
const safeResponse = truncateDiagnostic(redactOperatorText(responseText || ""));
const responsePart = safeResponse ? ` response="${safeResponse}";` : "";
const replayPart = replayDiagnostics ? ` ${replayDiagnostics};` : "";
return `LinkedIn ${label} request failed (${status}): queryId=${queryId || "unknown"}; variable keys/order=${variableKeys || "none"}; template=${templateShape || "none"}; rendered variables=${redactGraphQLVariablesForDiagnostics(variables)};${responsePart}${replayPart} refresh LinkedIn Messaging to recapture the live contract if this shape is no longer accepted.`;
return `LinkedIn ${label} request failed (${status}): queryId=${queryId || "unknown"}; variable keys/order=${variableKeys || "none"}; template=${templateShape || "none"}; rendered variables=${redactGraphQLVariablesForDiagnostics(variables)};${omittedPart}${responsePart}${replayPart} refresh LinkedIn Messaging to recapture the live contract if this shape is no longer accepted.`;
}

function buildOperatorNextAction({ backendReady, accountReady, hasTrack, hasCsrf, hasContract, contractFresh, lastError, serviceUrl }) {
Expand Down
136 changes: 136 additions & 0 deletions chrome-extension/test_background.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,92 @@ async function testAC5i_manualSyncAttempt9ReplaysCapturedGraphQLMethodAndSafeHea
assert(!!findIngestCall(env), "POST /sync/ingest was reached after Attempt #9 replay");
}

async function testAC5j_manualSyncAttempt11OmitsCapturedFirstPagePagination() {
console.log("\nAC5j: MANUAL_SYNC Attempt #11 omits stale first-page pagination and normalizes message window variables");
const expectedConversationsVariables = "(query:(predicateUnions:List((conversationCategoryPredicate:(category:PRIMARY_INBOX)))),count:20,mailboxUrn:urn:li:fsd_profile:42)";
const env = buildEnv({
linkedinResponses: {
conversations: (url) => {
const variables = decodeURIComponent(new URL(url).searchParams.get("variables") || "");
if (variables.includes("SHOULD_NOT_APPEAR")) return { __error__: 400, __text__: "stale pagination replayed" };
if (variables !== expectedConversationsVariables) {
return { __error__: 400, __text__: `bad conversations variables ${variables}` };
}
return {
data: {
messengerConversationsBySyncToken: {
elements: [
{
entityUrn: "urn:li:msg_conversation:attempt11-1",
conversationName: null,
conversationParticipants: [],
},
],
metadata: {},
},
},
};
},
messages: (url) => {
const variables = decodeURIComponent(new URL(url).searchParams.get("variables") || "");
if (
variables.includes("SHOULD_NOT_APPEAR") ||
variables.includes("1111111111111") ||
variables.includes("countBefore:7") ||
variables.includes("countAfter:3")
) {
return { __error__: 400, __text__: "stale message window replayed" };
}
const expected = /^\(deliveredAt:\d{10,},conversationUrn:urn:li:msg_conversation:attempt11-1,countBefore:20,countAfter:0\)$/;
if (!expected.test(variables)) return { __error__: 400, __text__: `bad messages variables ${variables}` };
return { data: { messengerMessagesBySyncToken: { elements: [] } } };
},
},
});
env.storage.accountId = 1;
env.storage.xLiTrack = "SYNC_TRACK";
env.storage.csrfToken = "SYNC_CSRF";
loadBackground(env);

const headerListener = env.listeners.onSendHeaders[0];
const conversationsVariables = "(query:(predicateUnions:List((conversationCategoryPredicate:(category:PRIMARY_INBOX)))),count:50,mailboxUrn:urn:li:fsd_profile:captured,nextCursor:SHOULD_NOT_APPEAR)";
const messagesVariables = "(deliveredAt:1111111111111,conversationUrn:urn:li:msg_conversation:captured,countBefore:7,countAfter:3)";
await headerListener.fn({
method: "GET",
url:
"https://www.linkedin.com/voyager/api/voyagerMessagingGraphQL/graphql" +
`?queryId=messengerConversations.attempt11&variables=${encodeURIComponent(conversationsVariables)}`,
requestHeaders: [
{ name: "x-li-track", value: "SYNC_TRACK" },
{ name: "csrf-token", value: "SYNC_CSRF" },
],
});
await headerListener.fn({
method: "GET",
url:
"https://www.linkedin.com/voyager/api/voyagerMessagingGraphQL/graphql" +
`?queryId=messengerMessages.attempt11&variables=${encodeURIComponent(messagesVariables)}`,
requestHeaders: [
{ name: "x-li-track", value: "SYNC_TRACK" },
{ name: "csrf-token", value: "SYNC_CSRF" },
],
});

const contract = env.storage.messagingContract || {};
const cursorEntry = contract.conversationsVariablesTemplate?.find((v) => v.key === "nextCursor");
const countEntry = contract.conversationsVariablesTemplate?.find((v) => v.key === "count");
assert(cursorEntry?.source === "cursor" && cursorEntry?.optional === true, "nextCursor captured as optional first-page cursor metadata");
assert(countEntry?.source === "count" && countEntry.defaultValue === 20, "captured conversations count is normalized to first-page default metadata");
assert(!JSON.stringify(contract).includes("SHOULD_NOT_APPEAR"), "captured pagination value is not stored in contract");
assert(contract.messagesVariablesTemplate?.some((v) => v.key === "deliveredAt" && v.source === "now"), "deliveredAt captured as runtime timestamp metadata");
assert(contract.messagesVariablesTemplate?.some((v) => v.key === "countBefore" && v.source === "countBefore"), "countBefore captured as first-page count metadata");
assert(contract.messagesVariablesTemplate?.some((v) => v.key === "countAfter" && v.source === "countAfter" && v.defaultValue === 0), "countAfter captured as first-page zero lookahead metadata");

const resp = await env.chrome.runtime.sendMessage({ type: "MANUAL_SYNC" });
assert(resp.ok === true, `Attempt #11 first-page replay reaches ingest (got: ${JSON.stringify(resp)})`);
assert(!!findIngestCall(env), "POST /sync/ingest was reached after Attempt #11 first-page replay");
}

async function testAC5b_manualSyncIncludesBearerToken() {
console.log("\nAC5b: MANUAL_SYNC includes Authorization on /sync/ingest when apiToken configured");
const env = buildEnv();
Expand Down Expand Up @@ -1027,6 +1113,10 @@ async function testAC7b_messagingContractMessages() {
contract.messagesVariablesTemplate.some((v) => v.key === "conversationUrn" && v.source === "conversationUrn" && !Object.prototype.hasOwnProperty.call(v, "value")),
"conversationUrn stored as runtime placeholder"
);
assert(
contract.messagesVariablesTemplate.some((v) => v.key === "count" && v.source === "count" && v.defaultValue === 20),
"captured message count is normalized to first-page default metadata"
);
assert(
contract.messagesVariablesTemplate.some((v) => v.key === "createdBefore" && v.source === "now"),
"createdBefore stored as dynamic timestamp metadata"
Expand Down Expand Up @@ -1167,6 +1257,50 @@ async function testAC8f_conversationsHttp400IncludesRedactedShapeDiagnostics() {
assert(!ingestCall, "POST /sync/ingest was NOT called after LinkedIn HTTP 400");
}

async function testAC8g_attempt11Http400ReportsOmittedCursorDiagnostics() {
console.log("\nAC8g: Attempt #11 HTTP 400 diagnostics report omitted optional cursor without leaking it");
const staleCursorSentinel = "REDACTED_CURSOR_SENTINEL";
const env = buildEnv({
linkedinResponses: {
conversations: () => ({ __error__: 400, __text__: `nextCursor:${staleCursorSentinel}; unsupported first page` }),
},
});
env.storage.accountId = 1;
env.storage.csrfToken = "SYNC_CSRF";
env.storage.xLiTrack = "SYNC_TRACK";
env.storage.messagingContract = {
conversationsQueryId: "messengerConversations.attempt11diag",
messagesQueryId: "messengerMessages.attempt11diag",
conversationsVariablesShape: ["query", "count", "mailboxUrn", "nextCursor"],
messagesVariablesShape: ["deliveredAt", "conversationUrn", "countBefore", "countAfter"],
conversationsVariablesTemplate: [
{ key: "query", value: "(predicateUnions:List((conversationCategoryPredicate:(category:PRIMARY_INBOX))))" },
{ key: "count", source: "count", defaultValue: 20 },
{ key: "mailboxUrn", source: "mailboxUrn" },
{ key: "nextCursor", source: "cursor", optional: true },
],
messagesVariablesTemplate: [
{ key: "deliveredAt", source: "now" },
{ key: "conversationUrn", source: "conversationUrn" },
{ key: "countBefore", source: "countBefore", defaultValue: 20 },
{ key: "countAfter", source: "countAfter", defaultValue: 0 },
],
endpointPath: "/voyager/api/voyagerMessagingGraphQL/graphql",
capturedAt: new Date().toISOString(),
};
loadBackground(env);

const resp = await env.chrome.runtime.sendMessage({ type: "MANUAL_SYNC" });
assert(resp.ok === false, "sync response is not ok on Attempt #11 diagnostic HTTP 400");
const error = resp.error || "";
assert(error.includes("nextCursor{source=cursor,optional}"), "error template marks nextCursor as optional cursor metadata");
assert(error.includes("omitted optional variables=nextCursor"), "error reports omitted optional nextCursor");
assert(error.includes("rendered variables=(query:(predicateUnions:List((conversationCategoryPredicate:(category:PRIMARY_INBOX)))),count:20,mailboxUrn:[runtime-mailboxUrn])"), "rendered variables omit nextCursor for first-page replay");
assert(error.includes("nextCursor:[redacted]"), "response cursor value is redacted in diagnostics");
assert(!error.includes(staleCursorSentinel), "raw cursor-like sentinel is not leaked");
assert(!findIngestCall(env), "POST /sync/ingest was NOT called after Attempt #11 diagnostic HTTP 400");
}

async function testAC8_manualSyncFailsWithoutContract() {
console.log("\nAC8: MANUAL_SYNC fails visibly when messaging contract is missing");
const env = buildEnv();
Expand Down Expand Up @@ -1391,6 +1525,7 @@ async function main() {
await testAC5g_manualSyncReplaysQuotedAttempt7DynamicVariables();
await testAC5h_manualSyncAttempt8ReplaysFromLinkedInPageWithSafeHeaders();
await testAC5i_manualSyncAttempt9ReplaysCapturedGraphQLMethodAndSafeHeaders();
await testAC5j_manualSyncAttempt11OmitsCapturedFirstPagePagination();
await testAC5b_manualSyncIncludesBearerToken();
await testAC5c_extensionDirectionForMyMessages();
await testAC6_manualRefresh();
Expand All @@ -1403,6 +1538,7 @@ async function main() {
await testAC8c_manualSyncFailsWithoutCsrf();
await testAC8d_extensionNeverLogsCookiesOrCsrf();
await testAC8f_conversationsHttp400IncludesRedactedShapeDiagnostics();
await testAC8g_attempt11Http400ReportsOmittedCursorDiagnostics();
await testAC8e_manualSyncFailsWithLegacyShapeOnlyContract();
await testAC10_operatorStatusReadyForSync();
await testAC10d_operatorStatusReadyForMinimalNoCountContract();
Expand Down
Loading