Earlier target monitoring (#46)

* Start collecting data even before main page target is created.

* Don't use private API - create new session for existing target.

* Remove no longer needed ts-ignore

* Wait for the debugger to start on new context. This allows us catch web worker and service worker requests. Fixes #44

* Add integration tests for early calls and other edge-cases.

* Fix interestCohort API exclusion from API collection integration test.

crawler.js

@@ -162,6 +162,9 @@ async function getSiteData(context, url, {
     // new target is created we will miss some requests, API calls, etc.
     const cdpClient = await page.target().createCDPSession();
 
+    // without this, we will miss the initial request for the web worker or service worker file
+    await cdpClient.send('Target.setAutoAttach', {autoAttach: true, waitForDebuggerOnStart: true});
+
     const initPageTimer = createTimer();
     for (let collector of collectors) {
         try {

tests/integration/apiCollection.test.js

@@ -0,0 +1,93 @@
+const {crawler, APICallCollector} = require('../../main.js');
+const assert = require('assert');
+const breakpoints = require('../../collectors/APICalls/breakpoints.js');
+
+async function main() {
+
+    // we are testing all APIs that we are monitoring against our privacy test page for fingerprinting
+
+    const apiData = await crawler(new URL('https://privacy-test-pages.glitch.me/privacy-protections/fingerprinting/?run'), {
+        log: () => {},
+        collectors: [new APICallCollector()]
+    });
+
+    const apiCalls = apiData.data.apis.callStats['https://privacy-test-pages.glitch.me/privacy-protections/fingerprinting/helpers/tests.js'];
+
+    // known fingerprinting breakpoints that are not invoked by our test page
+    const knownMissing = [
+        "window.name",
+        "PerformanceTiming.prototype.navigationStart",
+        "Document.cookie getter",
+        "Document.cookie setter",
+        "Navigator.prototype.onLine",
+        "Navigator.prototype.keyboard",
+        "Navigator.prototype.presentation",
+        "Event.prototype.timeStamp",
+        "KeyboardEvent.prototype.code",
+        "KeyboardEvent.prototype.keyCode",
+        "Touch.prototype.force",
+        "Touch.prototype.radiusX",
+        "Touch.prototype.radiusY",
+        "Touch.prototype.rotationAngle",
+        "WheelEvent.prototype.deltaX",
+        "WheelEvent.prototype.deltaY",
+        "WheelEvent.prototype.deltaZ",
+        "DeviceOrientationEvent.prototype.alpha",
+        "DeviceOrientationEvent.prototype.beta",
+        "DeviceOrientationEvent.prototype.gamma",
+        "DeviceOrientationEvent.prototype.absolute",
+        "DeviceMotionEvent.prototype.acceleration",
+        "DeviceMotionEvent.prototype.accelerationIncludingGravity",
+        "DeviceMotionEvent.prototype.rotationRate",
+        "Animation.prototype.currentTime",
+        "Animation.prototype.startTime",
+        "Gyroscope.prototype.x",
+        "Gyroscope.prototype.y",
+        "Gyroscope.prototype.z",
+        // method calls
+        "Document.prototype.interestCohort",
+        'window.matchMedia("prefers-color-scheme")',
+        'HTMLCanvasElement.prototype.constructor',
+        'CanvasRenderingContext2D.prototype.isPointInPath',
+        'Date.prototype.getTime',
+        'WebGLRenderingContext.prototype.getExtension',
+        'AudioWorkletNode.prototype.constructor',
+        'SharedWorker.prototype.constructor',
+        'BroadcastChannel.prototype.constructor',
+        'TouchEvent.prototype.constructor',
+        'URL.createObjectURL',
+        'CSSStyleDeclaration.setProperty("fontFamily",…)',
+        'Element.prototype.getClientRects',
+        'Sensor.prototype.constructor',
+    ];
+
+    breakpoints.forEach(object => {
+        /**
+         * @type {string}
+         */
+        let prefix;
+
+        if (object.proto) {
+            prefix = object.proto + '.prototype.';
+        } else if (object.global) {
+            prefix = object.global + '.';
+        }
+
+        object.props.forEach(prop => {
+            const propName = prop.description || (prefix + prop.name);
+
+            if (!apiCalls[propName] && !knownMissing.includes(propName)) {
+                assert(false, `Missing ${propName} property read.`);
+            }
+        });
+        object.methods.forEach(method => {
+            const methodName = method.description || (prefix + method.name);
+
+            if (!apiCalls[methodName] && !knownMissing.includes(methodName)) {
+                assert(false, `Missing ${methodName} method call.`);
+            }
+        });
+    });
+}
+
+main();

tests/integration/requestCollection.test.js

@@ -0,0 +1,26 @@
+const {crawler, RequestCollector} = require('../../main.js');
+const assert = require('assert');
+
+async function main() {
+
+    const requestData = await crawler(new URL('https://privacy-test-pages.glitch.me/privacy-protections/request-blocking/?run'), {
+        log: () => {},
+        collectors: [new RequestCollector()]
+    });
+
+    // we are testing edge cases - requests that we missed in the past
+
+    const serviceWorkerRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/service-worker.js'));
+
+    assert(serviceWorkerRequest, 'Service worker request captured.');
+
+    const webWorkerRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/worker.js'));
+
+    assert(webWorkerRequest, 'Web worker request captured.');
+
+    const cspRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/csp'));
+
+    assert(cspRequest, 'CSP report request captured.');
+}
+
+main();