inspector: add network payload buffer size limits

By default, the total buffered payloads are limited to 100MB, and 5MB for each single request. The oldest unused requests will be evicted first when the buffer size limit has been exceeded. PR-URL: #60236 Refs: https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-enable Reviewed-By: Ryuhei Shima <[email protected]> Reviewed-By: Darshan Sen <[email protected]>

Author: legendecas

node.gyp

@@ -422,6 +422,7 @@
 'test/cctest/test_quic_tokens.cc',
  ],
 'node_cctest_inspector_sources': [
+'test/cctest/inspector/test_network_requests_buffer.cc',
 'test/cctest/inspector/test_node_protocol.cc',
 'test/cctest/test_inspector_socket.cc',
 'test/cctest/test_inspector_socket_server.cc',

src/inspector/network_agent.cc

@@ -25,6 +25,8 @@ using v8::Object;
 using v8::Uint8Array;
 using v8::Value;
 
+constexprsize_tkDefaultMaxTotalBufferSize = 100 * 1024 * 1024; // 100MB
+
 // Get a protocol string property from the object.
 Maybe<protocol::String> ObjectGetProtocolString(v8::Local<v8::Context> context,
  Local<Object> object,
@@ -245,7 +247,8 @@ NetworkAgent::NetworkAgent(
  : inspector_(inspector),
 v8_inspector_(v8_inspector),
  env_(env),
- network_resource_manager_(std::move(network_resource_manager)){
+ network_resource_manager_(std::move(network_resource_manager)),
+ requests_(kDefaultMaxTotalBufferSize){
  event_notifier_map_["requestWillBeSent"] = &NetworkAgent::requestWillBeSent;
  event_notifier_map_["responseReceived"] = &NetworkAgent::responseReceived;
  event_notifier_map_["loadingFailed"] = &NetworkAgent::loadingFailed;
@@ -328,8 +331,15 @@ void NetworkAgent::Wire(protocol::UberDispatcher* dispatcher){
 protocol::Network::Dispatcher::wire(dispatcher, this);
 }
 
-protocol::DispatchResponse NetworkAgent::enable(){
+protocol::DispatchResponse NetworkAgent::enable(
+ std::optional<int> in_maxTotalBufferSize,
+ std::optional<int> in_maxResourceBufferSize){
  inspector_->Enable();
+ requests_ = RequestsBuffer(
+ in_maxTotalBufferSize.value_or(kDefaultMaxTotalBufferSize));
+if (in_maxResourceBufferSize){
+ max_resource_buffer_size_ = *in_maxResourceBufferSize;
+ }
 returnprotocol::DispatchResponse::Success();
 }
 
@@ -340,7 +350,7 @@ protocol::DispatchResponse NetworkAgent::disable(){
 
 protocol::DispatchResponse NetworkAgent::getRequestPostData(
 const protocol::String& in_requestId, protocol::String* out_postData){
-auto request_entry = requests_.find(in_requestId);
+auto request_entry = requests_.cfind(in_requestId);
 if (request_entry == requests_.end()){
 // Request not found, ignore it.
 returnprotocol::DispatchResponse::InvalidParams("Request not found");
@@ -361,7 +371,7 @@ protocol::DispatchResponse NetworkAgent::getRequestPostData(
 
 // Concat response bodies.
  protocol::Binary buf =
-protocol::Binary::concat(request_entry->second.request_data_blobs);
+protocol::Binary::concat(request_entry->second.request_data_blobs());
  *out_postData = protocol::StringUtil::fromUTF8(buf.data(), buf.size());
 returnprotocol::DispatchResponse::Success();
 }
@@ -370,7 +380,7 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
 const protocol::String& in_requestId,
  protocol::String* out_body,
 bool* out_base64Encoded){
-auto request_entry = requests_.find(in_requestId);
+auto request_entry = requests_.cfind(in_requestId);
 if (request_entry == requests_.end()){
 // Request not found, ignore it.
 returnprotocol::DispatchResponse::InvalidParams("Request not found");
@@ -390,7 +400,7 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
 
 // Concat response bodies.
  protocol::Binary buf =
-protocol::Binary::concat(request_entry->second.response_data_blobs);
+protocol::Binary::concat(request_entry->second.response_data_blobs());
 if (request_entry->second.response_charset == Charset::kBinary){
 // If the response is binary, we return base64 encoded data.
  *out_body = buf.toBase64();
@@ -409,22 +419,26 @@ protocol::DispatchResponse NetworkAgent::getResponseBody(
 
 protocol::DispatchResponse NetworkAgent::streamResourceContent(
 const protocol::String& in_requestId, protocol::Binary* out_bufferedData){
-auto it = requests_.find(in_requestId);
-if (it == requests_.end()){
-// Request not found, ignore it.
-returnprotocol::DispatchResponse::InvalidParams("Request not found");
- }
-auto& request_entry = it->second;
+bool is_response_finished = false;
+{
+auto it = requests_.find(in_requestId);
+if (it == requests_.end()){
+// Request not found, ignore it.
+returnprotocol::DispatchResponse::InvalidParams("Request not found");
+ }
+auto& request_entry = it->second;
 
- request_entry.is_streaming = true;
+request_entry.is_streaming = true;
 
-// Concat response bodies.
- *out_bufferedData =
-protocol::Binary::concat(request_entry.response_data_blobs);
-// Clear buffered data.
- request_entry.response_data_blobs.clear();
+// Concat response bodies.
+ *out_bufferedData =
+protocol::Binary::concat(request_entry.response_data_blobs());
+// Clear buffered data.
+ request_entry.clear_response_data_blobs();
+ is_response_finished = request_entry.is_response_finished;
+ }
 
-if (request_entry.is_response_finished){
+if (is_response_finished){
 // If the request is finished, remove the entry.
  requests_.erase(in_requestId);
  }
@@ -499,9 +513,11 @@ void NetworkAgent::requestWillBeSent(v8::Local<v8::Context> context,
  }
 
 auto request_charset = charset == "utf-8" ? Charset::kUTF8 : Charset::kBinary;
- requests_.emplace(
- request_id,
-RequestEntry(timestamp, request_charset, request->getHasPostData()));
+ requests_.emplace(request_id,
+RequestEntry(timestamp,
+ request_charset,
+ request->getHasPostData(),
+ max_resource_buffer_size_));
  frontend_->requestWillBeSent(request_id,
 std::move(request),
 std::move(initiator),
@@ -579,7 +595,7 @@ void NetworkAgent::loadingFinished(v8::Local<v8::Context> context,
 
  frontend_->loadingFinished(request_id, timestamp);
 
-auto request_entry = requests_.find(request_id);
+auto request_entry = requests_.cfind(request_id);
 if (request_entry == requests_.end()){
 // No entry found. Ignore it.
 return;
@@ -589,7 +605,7 @@ void NetworkAgent::loadingFinished(v8::Local<v8::Context> context,
 // Streaming finished, remove the entry.
  requests_.erase(request_id);
  } else{
-request_entry->second.is_response_finished = true;
+requests_.find(request_id)->second.is_response_finished = true;
  }
 }
 
@@ -630,7 +646,7 @@ void NetworkAgent::dataSent(v8::Local<v8::Context> context,
  }
  Local<Uint8Array> data = data_obj.As<Uint8Array>();
 auto data_bin = protocol::Binary::fromUint8Array(data);
- request_entry->second.request_data_blobs.push_back(data_bin);
+ request_entry->second.push_request_data_blob(data_bin);
 }
 
 voidNetworkAgent::dataReceived(v8::Local<v8::Context> context,
@@ -672,7 +688,7 @@ void NetworkAgent::dataReceived(v8::Local<v8::Context> context,
  frontend_->dataReceived(
  request_id, timestamp, data_length, encoded_data_length, data_bin);
  } else{
- request_entry.response_data_blobs.push_back(data_bin);
+ request_entry.push_response_data_blob(data_bin);
  }
 }
 

src/inspector/network_agent.h

@@ -3,6 +3,7 @@
 
 #include"env.h"
 #include"io_agent.h"
+#include"network_requests_buffer.h"
 #include"network_resource_manager.h"
 #include"node/inspector/protocol/Network.h"
 
@@ -15,31 +16,6 @@ namespace inspector{
 
 classNetworkInspector;
 
-// Supported charsets for devtools frontend on request/response data.
-// If the charset is kUTF8, the data is expected to be text and can be
-// formatted on the frontend.
-enumclassCharset{
-kUTF8,
-kBinary,
-};
-
-structRequestEntry{
-double timestamp;
-bool is_request_finished = false;
-bool is_response_finished = false;
-bool is_streaming = false;
- Charset request_charset;
- std::vector<protocol::Binary> request_data_blobs;
- Charset response_charset;
- std::vector<protocol::Binary> response_data_blobs;
-
-RequestEntry(double timestamp, Charset request_charset, bool has_request_body)
- : timestamp(timestamp),
-is_request_finished(!has_request_body),
- request_charset(request_charset),
- response_charset(Charset::kBinary){}
-};
-
 classNetworkAgent : publicprotocol::Network::Backend{
 public:
 explicitNetworkAgent(
@@ -50,7 +26,9 @@ class NetworkAgent : public protocol::Network::Backend{
 
 voidWire(protocol::UberDispatcher* dispatcher);
 
- protocol::DispatchResponse enable() override;
+ protocol::DispatchResponse enable(
+ std::optional<int> in_maxTotalBufferSize,
+ std::optional<int> in_maxResourceBufferSize) override;
 
  protocol::DispatchResponse disable() override;
 
@@ -104,12 +82,17 @@ class NetworkAgent : public protocol::Network::Backend{
  NetworkInspector* inspector_;
  v8_inspector::V8Inspector* v8_inspector_;
  std::shared_ptr<protocol::Network::Frontend> frontend_;
+
 using EventNotifier = void (NetworkAgent::*)(v8::Local<v8::Context> context,
  v8::Local<v8::Object>);
  std::unordered_map<protocol::String, EventNotifier> event_notifier_map_;
- std::map<protocol::String, RequestEntry> requests_;
  Environment* env_;
  std::shared_ptr<NetworkResourceManager> network_resource_manager_;
+
+// Per-resource buffer size in bytes to use when preserving network payloads
+// (XHRs, etc).
+size_t max_resource_buffer_size_ = 5 * 1024 * 1024; // 5MB
+ RequestsBuffer requests_;
 };
 
 } // namespace inspector