Feat/network

front_end/ndb.json

@@ -1,15 +1,21 @@
 {
- "modules" : [
- { "name": "ndb_sdk", "type": "autostart" },
- { "name": "ndb", "type": "autostart" },
- { "name": "layer_viewer" },
- { "name": "timeline_model" },
- { "name": "timeline" },
- { "name": "product_registry" },
- { "name": "mobile_throttling" },
- { "name": "ndb_ui" },
- { "name": "xterm" }
- ],
+ "modules": [
+ { "name": "ndb_sdk", "type": "autostart" },
+ { "name": "ndb", "type": "autostart" },
+ { "name": "layer_viewer" },
+ { "name": "timeline_model" },
+ { "name": "timeline" },
+ { "name": "product_registry" },
+ { "name": "mobile_throttling" },
+ { "name": "ndb_ui" },
+ { "name": "xterm" },
+ { "name": "emulation", "type": "autostart" },
+ { "name": "inspector_main", "type": "autostart" },
+ { "name": "mobile_throttling", "type": "autostart" },
+ { "name": "cookie_table" },
+ { "name": "har_importer" },
+ { "name": "network" }
+ ],
  "extends": "shell",
  "has_html": true
 }

front_end/ndb/NdbMain.js

@@ -172,9 +172,39 @@ Ndb.NodeProcessManager = class extends Common.Object {
  static async create(targetManager) {
  const manager = new Ndb.NodeProcessManager(targetManager);
  manager._service = await Ndb.backend.createService('ndd_service.js', rpc.handle(manager));
+ InspectorFrontendHost.sendMessageToBackend = manager.sendMessageToBackend.bind(manager);
  return manager;
  }
 
+ async sendMessageToBackend(message) {
+ if (this._service && this._service.sendMessage)
+ return this._service.sendMessage(message);
+ }
+
+ sendLoadingFinished({ type, payload }) {
+ SDK._mainConnection._onMessage(JSON.stringify({
+ method: 'Network.loadingFinished',
+ params: payload
+ }));
+ }
+
+ responseToFrontEnd(id, result) {
+ InspectorFrontendHost.events.dispatchEventToListeners(
+ InspectorFrontendHostAPI.Events.DispatchMessage,
+ {
+ id,
+ result
+ }
+ );
+ }
+
+ sendNetworkData({ type, payload }) {
+ SDK._mainConnection._onMessage(JSON.stringify({
+ method: type,
+ params: payload
+ }));
+ }
+
  env() {
  return this._service.env();
  }
@@ -208,6 +238,26 @@ Ndb.NodeProcessManager = class extends Common.Object {
  info.id, userFriendlyName(info), SDK.Target.Type.Node,
  this._targetManager.targetById(info.ppid) || this._targetManager.mainTarget(), undefined, false, connection);
  target[NdbSdk.connectionSymbol] = connection;
+
+ try {
+ // this doesnt work
+ // target.runtimeAgent().invoke_evaluate({
+ // expression: `
+ // const zlib = require('http');
+ // console.log('foo');
+ // `
+ // });
+ //
+ // but this does
+ // target.runtimeAgent().invoke_evaluate({
+ // expression: `
+ // console.log('foo');
+ // `
+ // });
+ } catch(err) {
+ console.log(err);
+ }
+
  await this.addFileSystem(info.cwd, info.scriptName);
  if (info.scriptName) {
  const scriptURL = Common.ParsedURL.platformPathToURL(info.scriptName);

front_end/ndb/module.json

@@ -52,7 +52,14 @@
  "className": "Ndb.ContextMenuProvider"
  }
  ],
- "dependencies": ["common", "sdk", "ndb_sdk", "bindings", "persistence", "components"],
+ "dependencies": [
+ "common",
+ "sdk",
+ "ndb_sdk",
+ "bindings",
+ "persistence",
+ "components"
+ ],
  "scripts": [
  "InspectorFrontendHostOverrides.js",
  "Connection.js",

lib/backend.js

@@ -48,6 +48,12 @@ class Backend {
  opn(url);
  }
 
+ async httpMonkeyPatchingSource() {
+ const pathToHttpPatch = path.resolve(__dirname, '..', './lib/preload/ndb/httpMonkeyPatching.js');
+ const content = await fsReadFile(pathToHttpPatch, 'utf8');
+ return content;
+ }
+
  pkg() {
  // TODO(ak239spb): implement it as decorations over package.json file.
  try {

lib/preload/ndb/httpMonkeyPatching.js

@@ -0,0 +1,153 @@
+const zlib = require('zlib');
+const http = require('http');
+const https = require('https');
+
+const initTime = process.hrtime();
+
+// DT requires us to use relative time in a strange format (xxx.xxx)
+const getTime = () => {
+ const diff = process.hrtime(initTime);
+
+ return diff[0] + diff[1] / 1e9;
+};
+
+const formatRequestHeaders = req => {
+ if (!req.headers) return {};
+ return Object.keys(req.headers).reduce((acc, k) => {
+ if (typeof req.headers[k] === 'string') acc[k] = req.headers[k];
+ return acc;
+ }, {});
+};
+
+const formatResponseHeaders = res => {
+ if (!res.headers) return {};
+ return Object.keys(res.headers).reduce((acc, k) => {
+ if (typeof res.headers[k] === 'string') acc[k] = res.headers[k];
+ return acc;
+ }, {});
+};
+
+const getMineType = mimeType => {
+ // nasty hack for ASF
+ if (mimeType === 'OPENJSON')
+ return 'application/json;charset=UTF-8';
+
+
+ return mimeType;
+};
+
+const cacheRequests = {};
+let id = 1;
+const getId = () => id++;
+
+const callbackWrapper = (callback, req) => res => {
+ const requestId = getId();
+ res.req.__requestId = requestId;
+
+ process.send({
+ payload: {
+ requestId: requestId,
+ loaderId: requestId,
+ documentURL: req.href,
+ request: {
+ url: req.href,
+ method: req.method,
+ headers: formatRequestHeaders(req),
+ mixedContentType: 'none',
+ initialPriority: 'VeryHigh',
+ referrerPolicy: 'no-referrer-when-downgrade',
+ postData: req.body
+ },
+ timestamp: getTime(),
+ wallTime: Date.now(),
+ initiator: {
+ type: 'other'
+ },
+ type: 'Document'
+ },
+ type: 'Network.requestWillBeSent'
+ });
+
+ const encoding = res.headers['content-encoding'];
+ let rawData = [];
+
+ const onEnd = function() {
+ rawData = Buffer.concat(rawData);
+ rawData = rawData.toString('base64');
+
+ cacheRequests[res.req.__requestId] = {
+ ...res,
+ __rawData: rawData,
+ base64Encoded: true
+ };
+ const payload = {
+ id: res.req.__requestId,
+ requestId: res.req.__requestId,
+ loaderId: res.req.__requestId,
+ base64Encoded: true,
+ data: cacheRequests[res.req.__requestId].__rawData,
+ timestamp: getTime(),
+ type: 'XHR',
+ encodedDataLength: 100,
+ response: {
+ url: req.href,
+ status: res.statusCode,
+ statusText: res.statusText,
+ // set-cookie prop in the header has value as an array
+ // for example: ["__cfduid=dbfe006ef71658bf4dba321343c227f9a15449556…20:29 GMT; path=/; domain=.typicode.com; HttpOnly"]
+ headers: formatResponseHeaders(res),
+ mimeType: getMineType(
+ res.headers['content-encoding'] ||
+ res.headers['content-type']
+ ),
+ requestHeaders: formatRequestHeaders(req)
+ }
+ };
+
+ // Send the response back.
+ process.send({ payload: payload, type: 'Network.responseReceived' });
+ process.send({ payload: payload, type: 'Network.loadingFinished' });
+ };
+
+ if (encoding === 'gzip' || encoding === 'x-gzip') {
+ const gunzip = zlib.createGunzip();
+ res.pipe(gunzip);
+
+ gunzip.on('data', function(data) {
+ rawData.push(data);
+ });
+ gunzip.on('end', onEnd);
+ } else {
+ res.on('data', chunk => {
+ rawData.push(chunk);
+ });
+ res.on('end', onEnd);
+ }
+
+ callback && callback(res);
+};
+
+const originHTTPRequest = http.request;
+http.request = function wrapMethodRequest(req, callback) {
+ const request = originHTTPRequest.call(
+ this,
+ req,
+ callbackWrapper(callback, req)
+ );
+ return request;
+};
+
+const originHTTPSRequest = https.request;
+https.request = function wrapMethodRequest(req, callback) {
+ const request = originHTTPSRequest.call(
+ this,
+ req,
+ callbackWrapper(callback, req)
+ );
+ const originWrite = request.write.bind(request);
+ request.write = data => {
+ req.body = data.toString();
+ originWrite(data);
+ };
+ return request;
+};

services/ndd_service.js

@@ -23,6 +23,8 @@ function silentRpcErrors(error) {
 process.on('uncaughtException', silentRpcErrors);
 process.on('unhandledRejection', silentRpcErrors);
 
+const catchedRequests = {};
+
 const DebugState = {
  WS_OPEN: 1,
  WS_ERROR: 2,
@@ -141,17 +143,31 @@ class NddService {
  };
  }
 
+ sendMessage(rawMessage) {
+ const message = JSON.parse(rawMessage);
+ // send message to frontend directly
+ // (eg: getResponseBody)
+ const { base64Encoded, data } = catchedRequests[message.params.requestId];
+ this._frontend.responseToFrontEnd(message.id, { base64Encoded, body: data });
+ }
+
  async debug(execPath, args, options) {
  const env = this.env();
  if (options.data)
  env.NDD_DATA = options.data;
+
  const p = spawn(execPath, args, {
  cwd: options.cwd,
  env: { ...process.env, ...env },
- stdio: options.ignoreOutput ? 'ignore' : ['inherit', 'pipe', 'pipe'],
+ stdio: options.ignoreOutput ? ['ignore', 'ignore', 'ignore', 'ipc'] : ['pipe', 'pipe', 'pipe', 'ipc'],
  windowsHide: true
  });
  if (!options.ignoreOutput) {
+ p.on('message', ({ type, payload }) => {
+ if (!(type && payload)) return;
+ catchedRequests[payload.id] = payload;
+ this._frontend.sendNetworkData({ type, payload });
+ });
  p.stderr.on('data', data => {
  if (process.connected)
  this._frontend.terminalData('stderr', data.toString('base64'));