add logging

.evergreen/config.in.yml

@@ -107,6 +107,20 @@ functions:
         binary: bash
         args:
           - .evergreen/run-tests.sh
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/connection-logs.txt
+        optional: true
+        # Upload the coverage report for all tasks in a single build to the same directory.
+        # TODO NODE-4707 - change upload directory to ${UPLOAD_BUCKET}
+        # This change will require changing the `download and merge coverage` func as well
+        remote_file: mongo-node-driver/${revision}/${version_id}/${build_variant}/${task_name}/connection-logs.txt
+        bucket: mciuploads
+        permissions: public-read
+        content_type: text/plain
+        display_name: "Connection Logs"
 
   "run serverless tests":
     - command: timeout.update
@@ -988,6 +1002,20 @@ task_groups:
             - .evergreen/setup-serverless.sh
 
     teardown_group:
+      - command: s3.put
+        params:
+          aws_key: ${aws_key}
+          aws_secret: ${aws_secret}
+          local_file: src/connection-logs.txt
+          optional: true
+          # Upload the coverage report for all tasks in a single build to the same directory.
+          # TODO NODE-4707 - change upload directory to ${UPLOAD_BUCKET}
+          # This change will require changing the `download and merge coverage` func as well
+          remote_file: mongo-node-driver/${revision}/${version_id}/${build_variant}/${task_name}/connection-logs.txt
+          bucket: mciuploads
+          permissions: public-read
+          content_type: text/plain
+          display_name: "Connection Logs"
       - func: "upload test results"
       - command: subprocess.exec
         params:

.evergreen/config.yml

@@ -80,6 +80,17 @@ functions:
         binary: bash
         args:
           - .evergreen/run-tests.sh
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/connection-logs.txt
+        optional: true
+        remote_file: mongo-node-driver/${revision}/${version_id}/${build_variant}/${task_name}/connection-logs.txt
+        bucket: mciuploads
+        permissions: public-read
+        content_type: text/plain
+        display_name: Connection Logs
   run serverless tests:
     - command: timeout.update
       params:
@@ -2874,6 +2885,17 @@ task_groups:
           args:
             - .evergreen/setup-serverless.sh
     teardown_group:
+      - command: s3.put
+        params:
+          aws_key: ${aws_key}
+          aws_secret: ${aws_secret}
+          local_file: src/connection-logs.txt
+          optional: true
+          remote_file: mongo-node-driver/${revision}/${version_id}/${build_variant}/${task_name}/connection-logs.txt
+          bucket: mciuploads
+          permissions: public-read
+          content_type: text/plain
+          display_name: Connection Logs
       - func: upload test results
       - command: subprocess.exec
         params:

src/cmap/connect.ts

@@ -391,7 +391,12 @@ export async function makeSocket(options: MakeConnectionOptions): Promise<Stream
     socket
       .once(connectEvent, () => resolve(socket))
       .once('error', cause =>
-        reject(new MongoNetworkError(MongoError.buildErrorMessage(cause), { cause }))
+        reject(
+          new MongoNetworkError(
+            MongoError.buildErrorMessage(cause) + '- error encountered while establishing socket',
+            { cause }
+          )
+        )
       )
       .once('timeout', () => {
         reject(
@@ -489,7 +494,10 @@ async function makeSocks5Connection(options: MakeConnectionOptions): Promise<Str
     });
     existingSocket = connection.socket;
   } catch (cause) {
-    throw new MongoNetworkError(MongoError.buildErrorMessage(cause), { cause });
+    throw new MongoNetworkError(
+      MongoError.buildErrorMessage(cause) + '- during socks5 proxy establishment',
+      { cause }
+    );
   }
 
   // Finally, now treat the resulting duplex stream as the

src/cmap/connection.ts

@@ -1,3 +1,4 @@
+import { createWriteStream } from 'fs';
 import { type Readable, Transform, type TransformCallback } from 'stream';
 import { clearTimeout, setTimeout } from 'timers';
 
@@ -6,6 +7,7 @@ import {
   deserialize,
   type DeserializeOptions,
   type Document,
+  EJSON,
   type ObjectId
 } from '../bson';
 import { type AutoEncrypter } from '../client-side-encryption/auto_encrypter';
@@ -179,6 +181,45 @@ function streamIdentifier(stream: Stream, options: ConnectionOptions): string {
   return uuidV4().toString('hex');
 }
 
+export const logger = createWriteStream('connection-logs.txt');
+export const write = (payload: Document) => {
+  payload.timestamp = new Date().toISOString();
+  payload.hostname = process.env.MONGODB_URI;
+  payload.ip = process.env.IP;
+
+  const log = JSON.stringify(payload);
+  logger.write(log);
+  logger.write('\n');
+};
+
+const writeEvent =
+  (event: string) =>
+  <T extends Document>(payload: T) => {
+    (payload as T & { event: string }).event = event;
+    write(payload);
+  };
+
+export const writeStarted = writeEvent('commandStarted')<{
+  requestId: number;
+  connectionId: number | '<monitor>';
+}>;
+
+export const readStarted = writeEvent('readStarted')<{
+  requestId: number;
+  connectionId: number | '<monitor>';
+}>;
+
+export const readSucceeded = writeEvent('readSucceeded')<{
+  requestId: number;
+  connectionId: number | '<monitor>';
+}>;
+
+export const readFailed = writeEvent('readFailed')<{
+  requestId: number;
+  connectionId: number | '<monitor>';
+  error: Error;
+}>;
+
 /** @internal */
 export class Connection extends TypedEventEmitter<ConnectionEvents> {
   public id: number | '<monitor>';
@@ -451,7 +492,13 @@ export class Connection extends TypedEventEmitter<ConnectionEvents> {
       this.socketTimeoutMS;
     this.socket.setTimeout(timeout);
 
+    const payload = {
+      connectionId: this.id,
+      requestId: message.requestId
+    };
     try {
+      writeStarted(payload);
+
       await this.writeCommand(message, {
         agreedCompressor: this.description.compressor ?? 'none',
         zlibCompressionLevel: this.description.zlibCompressionLevel,
@@ -476,16 +523,28 @@ export class Connection extends TypedEventEmitter<ConnectionEvents> {
         );
       }
 
-      for await (const response of this.readMany(options)) {
-        this.socket.setTimeout(0);
-        const bson = response.parse();
+      readStarted(payload);
+      try {
+        for await (const response of this.readMany(options)) {
+          readSucceeded(payload);
+          this.socket.setTimeout(0);
+          const bson = response.parse();
 
-        const document = (responseType ?? MongoDBResponse).make(bson);
+          const document = (responseType ?? MongoDBResponse).make(bson);
 
-        yield document;
-        this.throwIfAborted();
+          yield document;
 
-        this.socket.setTimeout(timeout);
+          readStarted(payload);
+          this.throwIfAborted();
+
+          this.socket.setTimeout(timeout);
+        }
+      } catch (error) {
+        readFailed({
+          ...payload,
+          error
+        });
+        throw error;
       }
     } finally {
       this.socket.setTimeout(0);

src/error.ts

@@ -1045,8 +1045,16 @@ export class MongoNetworkError extends MongoError {
    * @public
    **/
   constructor(message: string, options?: MongoNetworkErrorOptions) {
+    message = message + `(timestamp = ${new Date().toISOString()})`;
     super(message, { cause: options?.cause });
     this.beforeHandshake = !!options?.beforeHandshake;
+
+    const { write } = require('./cmap/connection');
+    write({
+      event: 'network error',
+      message,
+      error: options?.cause
+    });
   }
 
   override get name(): string {

test/tools/runner/hooks/configuration.ts

@@ -22,6 +22,7 @@ import { NodeVersionFilter } from '../filters/node_version_filter';
 import { OSFilter } from '../filters/os_filter';
 import { ServerlessFilter } from '../filters/serverless_filter';
 import { type Filter } from '../filters/filter';
+import { spawnSync } from 'child_process';
 
 // Default our tests to have auth enabled
 // A better solution will be tackled in NODE-3714
@@ -46,6 +47,9 @@ const loadBalanced = SINGLE_MONGOS_LB_URI && MULTI_MONGOS_LB_URI;
 const filters: Filter[] = [];
 
 let initializedFilters = false;
+
+process.env.IP = spawnSync('curl', ['ifconfig.me'], { encoding: 'utf-8' }).stdout;
+
 async function initializeFilters(client): Promise<Record<string, any>> {
   if (initializedFilters) {
     return {};