diff --git a/FiddlerImportNetlog/FiddlerInterface.cs b/FiddlerImportNetlog/FiddlerInterface.cs index 9f7b567..b4f618d 100644 --- a/FiddlerImportNetlog/FiddlerInterface.cs +++ b/FiddlerImportNetlog/FiddlerInterface.cs @@ -8,7 +8,7 @@ namespace FiddlerImportNetlog { - [ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.3). See https://dev.chromium.org/for-testers/providing-network-details for more details.")] + [ProfferFormat("NetLog JSON", "Chromium's JSON-based event log format (v1.3.1). See https://dev.chromium.org/for-testers/providing-network-details for more details.")] public class HTTPArchiveFormatImport : ISessionImporter { public Session[] ImportSessions(string sFormat, Dictionary dictOptions, EventHandler evtProgressNotifications) @@ -48,7 +48,7 @@ public Session[] ImportSessions(string sFormat, Dictionary dictO else { Stream oFS = File.OpenRead(sFilename); - + FiddlerApplication.Log.LogFormat("!NetLog Importer is loading {0}", sFilename); // Check to see if this file data was GZIP'd or PKZIP'd. bool bWasGZIP = false; bool bWasPKZIP = false; diff --git a/FiddlerImportNetlog/Importer.cs b/FiddlerImportNetlog/Importer.cs index a3bad74..c0cd38d 100644 --- a/FiddlerImportNetlog/Importer.cs +++ b/FiddlerImportNetlog/Importer.cs @@ -31,12 +31,14 @@ struct Magics public int SEND_QUIC_HEADERS; public int SEND_HTTP2_HEADERS; public int READ_HEADERS; + public int FAKE_RESPONSE_HEADERS_CREATED; public int COOKIE_INCLUSION_STATUS; public int FILTERED_BYTES_READ; public int SEND_BODY; public int SEND_REQUEST; public int SSL_CERTIFICATES_RECEIVED; public int SSL_HANDSHAKE_MESSAGE_RECEIVED; + public int TCP_CONNECT; } internal static string DescribeExceptionWithStack(Exception eX) @@ -151,12 +153,14 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents) NetLogMagics.SEND_QUIC_HEADERS = 6; NetLogMagics.SEND_HTTP2_HEADERS = 7; NetLogMagics.READ_HEADERS = 8; - NetLogMagics.COOKIE_INCLUSION_STATUS = 9; - NetLogMagics.FILTERED_BYTES_READ = 10; - NetLogMagics.SEND_BODY = 11; - NetLogMagics.SEND_REQUEST = 12; - NetLogMagics.SSL_CERTIFICATES_RECEIVED = 13; - NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = 14; + NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = 9; + NetLogMagics.COOKIE_INCLUSION_STATUS = 10; + NetLogMagics.FILTERED_BYTES_READ = 11; + NetLogMagics.SEND_BODY = 12; + NetLogMagics.SEND_REQUEST = 13; + NetLogMagics.SSL_CERTIFICATES_RECEIVED = 14; + NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = 15; + NetLogMagics.TCP_CONNECT = 16; List listEvents = new List(); foreach (Hashtable htItem in alTraceEvents) @@ -196,14 +200,14 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents) iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) continue; // Get (or create) the List of entries for this SOCKET. - if (!dictSecureSockets.ContainsKey(iSocketID)) + if (!dictSockets.ContainsKey(iSocketID)) { events = new List(); - dictSecureSockets.Add(iSocketID, events); + dictSockets.Add(iSocketID, events); } else { - events = dictSecureSockets[iSocketID]; + events = dictSockets[iSocketID]; } // Add this event to the SOCKET's list. events.Add(htEvent); @@ -270,7 +274,7 @@ private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents) sessSummary.utilSetResponseBody(sbClientInfo.ToString());*/ //GenerateDebugTreeSession(dictURLRequests); - //GenerateSocketListSession(dictSecureSockets); + //GenerateSocketListSession(dictSockets); NotifyProgress(1, "Import Completed."); } @@ -328,14 +332,16 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) NetLogMagics.SEND_QUIC_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS"], -995); NetLogMagics.SEND_HTTP2_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_HTTP2_SEND_REQUEST_HEADERS"], -994); NetLogMagics.READ_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_READ_RESPONSE_HEADERS"], -993); - NetLogMagics.FILTERED_BYTES_READ = getIntValue(htEventTypes["URL_REQUEST_JOB_FILTERED_BYTES_READ"], -992); - NetLogMagics.COOKIE_INCLUSION_STATUS = getIntValue(htEventTypes["COOKIE_INCLUSION_STATUS"], -991); - NetLogMagics.SEND_BODY = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_BODY"], -990); - NetLogMagics.SEND_REQUEST = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST"], -989); + NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = getIntValue(htEventTypes["URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED"], -992); + NetLogMagics.FILTERED_BYTES_READ = getIntValue(htEventTypes["URL_REQUEST_JOB_FILTERED_BYTES_READ"], -991); + NetLogMagics.COOKIE_INCLUSION_STATUS = getIntValue(htEventTypes["COOKIE_INCLUSION_STATUS"], -990); + NetLogMagics.SEND_BODY = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_BODY"], -989); + NetLogMagics.SEND_REQUEST = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST"], -988); // Socket-level Events - NetLogMagics.SSL_CERTIFICATES_RECEIVED = getIntValue(htEventTypes["SSL_CERTIFICATES_RECEIVED"], -988); - NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_RECEIVED"], -987); + NetLogMagics.SSL_CERTIFICATES_RECEIVED = getIntValue(htEventTypes["SSL_CERTIFICATES_RECEIVED"], -987); + NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_RECEIVED"], -986); + NetLogMagics.TCP_CONNECT = getIntValue(htEventTypes["TCP_CONNECT"], -987); // Get ALL event type names as strings for pretty print view dictEventTypes = new Dictionary(); @@ -410,7 +416,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) int iEvent = -1; int iLastPct = 25; var dictURLRequests = new Dictionary>(); - var dictSecureSockets = new Dictionary>(); + var dictSockets = new Dictionary>(); // Loop over events; bucket those associated to URLRequests by the source request's ID. ArrayList alEvents = htFile["events"] as ArrayList; @@ -435,17 +441,18 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) int iSocketID = getIntValue(htSource["id"], -1); if (iType != NetLogMagics.SSL_CERTIFICATES_RECEIVED && - iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) continue; + iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED && + iType != NetLogMagics.TCP_CONNECT) continue; // Get (or create) the List of entries for this SOCKET. - if (!dictSecureSockets.ContainsKey(iSocketID)) + if (!dictSockets.ContainsKey(iSocketID)) { events = new List(); - dictSecureSockets.Add(iSocketID, events); + dictSockets.Add(iSocketID, events); } else { - events = dictSecureSockets[iSocketID]; + events = dictSockets[iSocketID]; } // Add this event to the SOCKET's list. events.Add(htEvent); @@ -506,7 +513,7 @@ public bool ExtractSessionsFromJSON(Hashtable htFile) sessSummary.utilSetResponseBody(sbClientInfo.ToString()); GenerateDebugTreeSession(dictURLRequests); - GenerateSocketListSession(dictSecureSockets); + GenerateSocketListSession(dictSockets); NotifyProgress(1, "Import Completed."); return true; @@ -600,6 +607,24 @@ private void GenerateSocketListSession(Dictionary> dictSock int iType = getIntValue(htEvent["type"], -1); var htParams = (Hashtable) htEvent["params"]; + if (iType == NetLogMagics.TCP_CONNECT) + { + if (htParams.ContainsKey("local_address")) + { + htThisSocket.Add("local_address", htParams["local_address"]); + } + //"remote_address", "local_address", "address_list" + if (htParams.ContainsKey("remote_address")) + { + htThisSocket.Add("remote_address", htParams["remote_address"]); + } + if (htParams.ContainsKey("address_list")) + { + htThisSocket.Add("address_list", htParams["address_list"]); + } + continue; + } + if (iType == NetLogMagics.SSL_CERTIFICATES_RECEIVED) { StringBuilder sbCertsReceived = new StringBuilder(); @@ -730,7 +755,7 @@ private void GenerateSocketListSession(Dictionary> dictSock { _listSessions.Add(Session.BuildFromData(false, new HTTPRequestHeaders( - String.Format("/SECURE_SOCKETS"), // TODO: Add Machine name? + String.Format("/SOCKETS"), // TODO: Add Machine name? new[] { "Host: NETLOG" }), Utilities.emptyByteArray, new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), @@ -857,7 +882,7 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) // Most events we care about should have parameters. LANDMINE_MEME HERE if (iType != NetLogMagics.SEND_REQUEST && null == htParams) continue; - FiddlerApplication.Log.LogFormat("URLRequest#{0} - Event type: {1} - {2}", kvpUR.Key, iType, sURL); + // FiddlerApplication.Log.LogFormat("URLRequest#{0} - Event type: {1} - {2}", kvpUR.Key, iType, sURL); #region ParseImportantEvents // C# cannot |switch()| on non-constant case values. Hrmph. @@ -1032,7 +1057,8 @@ private void ParseSessionsFromBucket(KeyValuePair> kvpUR) continue; } - if (iType == NetLogMagics.READ_HEADERS) + if ((iType == NetLogMagics.READ_HEADERS) || + (iType == NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED)) { ArrayList alHeaderLines = htParams["headers"] as ArrayList; oTimers.ServerBeginResponse = oTimers.FiddlerGotResponseHeaders = GetTimeStamp(htEvent["time"], baseTime); diff --git a/FiddlerImportNetlog/Properties/AssemblyInfo.cs b/FiddlerImportNetlog/Properties/AssemblyInfo.cs index 66c7ed4..b89a3ac 100644 --- a/FiddlerImportNetlog/Properties/AssemblyInfo.cs +++ b/FiddlerImportNetlog/Properties/AssemblyInfo.cs @@ -6,9 +6,28 @@ [assembly: AssemblyCopyright("Copyright ©2021 Eric Lawrence")] [assembly: System.Resources.NeutralResourcesLanguage("en-US")] [assembly: ComVisible(false)] -[assembly: AssemblyVersion("1.3.0.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute in FiddlerInterface.cs to match! +[assembly: AssemblyVersion("1.3.1.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute in FiddlerInterface.cs to match! [assembly: Fiddler.RequiredVersion("4.6.0.0")] + +/* +TODO: +HTTP_STREAM_JOB has a binding between the request and the socket. Hook them up so we can propagate the connection info to the URL_REQUEST-generated Sessions. + +t=3262 [st=0] SOCKET_POOL_BOUND_TO_SOCKET + --> source_dependency = 1250 (SOCKET) +t=3262 [st=0] HTTP_STREAM_JOB_BOUND_TO_REQUEST + --> source_dependency = 1701 (URL_REQUEST) +*/ + +// v1.3.1.0 +// Support for FAKE_RESPONSE_HEADERS_CREATED for HSTS and Automatic HTTPS upgrades +// Add socket address info to generated SOCKETS list's session + +// v1.3.0.1 +// Less Log spam +// Write imported filename to log + // v1.3 // Support importing NetLog events from a Chromium trace json file