diff --git a/crawler.js b/crawler.js index 001b3b60..b2f1e1f1 100644 --- a/crawler.js +++ b/crawler.js @@ -162,6 +162,9 @@ async function getSiteData(context, url, { // new target is created we will miss some requests, API calls, etc. const cdpClient = await page.target().createCDPSession(); + // without this, we will miss the initial request for the web worker or service worker file + await cdpClient.send('Target.setAutoAttach', {autoAttach: true, waitForDebuggerOnStart: true}); + const initPageTimer = createTimer(); for (let collector of collectors) { try { diff --git a/tests/integration/apiCollection.test.js b/tests/integration/apiCollection.test.js new file mode 100644 index 00000000..a0b4f04f --- /dev/null +++ b/tests/integration/apiCollection.test.js @@ -0,0 +1,93 @@ +const {crawler, APICallCollector} = require('../../main.js'); +const assert = require('assert'); +const breakpoints = require('../../collectors/APICalls/breakpoints.js'); + +async function main() { + + // we are testing all APIs that we are monitoring against our privacy test page for fingerprinting + + const apiData = await crawler(new URL('https://privacy-test-pages.glitch.me/privacy-protections/fingerprinting/?run'), { + log: () => {}, + collectors: [new APICallCollector()] + }); + + const apiCalls = apiData.data.apis.callStats['https://privacy-test-pages.glitch.me/privacy-protections/fingerprinting/helpers/tests.js']; + + // known fingerprinting breakpoints that are not invoked by our test page + const knownMissing = [ + "window.name", + "PerformanceTiming.prototype.navigationStart", + "Document.cookie getter", + "Document.cookie setter", + "Navigator.prototype.onLine", + "Navigator.prototype.keyboard", + "Navigator.prototype.presentation", + "Event.prototype.timeStamp", + "KeyboardEvent.prototype.code", + "KeyboardEvent.prototype.keyCode", + "Touch.prototype.force", + "Touch.prototype.radiusX", + "Touch.prototype.radiusY", + "Touch.prototype.rotationAngle", + "WheelEvent.prototype.deltaX", + "WheelEvent.prototype.deltaY", + "WheelEvent.prototype.deltaZ", + "DeviceOrientationEvent.prototype.alpha", + "DeviceOrientationEvent.prototype.beta", + "DeviceOrientationEvent.prototype.gamma", + "DeviceOrientationEvent.prototype.absolute", + "DeviceMotionEvent.prototype.acceleration", + "DeviceMotionEvent.prototype.accelerationIncludingGravity", + "DeviceMotionEvent.prototype.rotationRate", + "Animation.prototype.currentTime", + "Animation.prototype.startTime", + "Gyroscope.prototype.x", + "Gyroscope.prototype.y", + "Gyroscope.prototype.z", + // method calls + "Document.prototype.interestCohort", + 'window.matchMedia("prefers-color-scheme")', + 'HTMLCanvasElement.prototype.constructor', + 'CanvasRenderingContext2D.prototype.isPointInPath', + 'Date.prototype.getTime', + 'WebGLRenderingContext.prototype.getExtension', + 'AudioWorkletNode.prototype.constructor', + 'SharedWorker.prototype.constructor', + 'BroadcastChannel.prototype.constructor', + 'TouchEvent.prototype.constructor', + 'URL.createObjectURL', + 'CSSStyleDeclaration.setProperty("fontFamily",…)', + 'Element.prototype.getClientRects', + 'Sensor.prototype.constructor', + ]; + + breakpoints.forEach(object => { + /** + * @type {string} + */ + let prefix; + + if (object.proto) { + prefix = object.proto + '.prototype.'; + } else if (object.global) { + prefix = object.global + '.'; + } + + object.props.forEach(prop => { + const propName = prop.description || (prefix + prop.name); + + if (!apiCalls[propName] && !knownMissing.includes(propName)) { + assert(false, `Missing ${propName} property read.`); + } + }); + object.methods.forEach(method => { + const methodName = method.description || (prefix + method.name); + + if (!apiCalls[methodName] && !knownMissing.includes(methodName)) { + assert(false, `Missing ${methodName} method call.`); + } + }); + }); +} + +main(); diff --git a/tests/integration/requestCollection.test.js b/tests/integration/requestCollection.test.js new file mode 100644 index 00000000..f79b696c --- /dev/null +++ b/tests/integration/requestCollection.test.js @@ -0,0 +1,26 @@ +const {crawler, RequestCollector} = require('../../main.js'); +const assert = require('assert'); + +async function main() { + + const requestData = await crawler(new URL('https://privacy-test-pages.glitch.me/privacy-protections/request-blocking/?run'), { + log: () => {}, + collectors: [new RequestCollector()] + }); + + // we are testing edge cases - requests that we missed in the past + + const serviceWorkerRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/service-worker.js')); + + assert(serviceWorkerRequest, 'Service worker request captured.'); + + const webWorkerRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/worker.js')); + + assert(webWorkerRequest, 'Web worker request captured.'); + + const cspRequest = requestData.data.requests.find((/** @type {{url: string}} **/ r) => r.url.endsWith('/csp')); + + assert(cspRequest, 'CSP report request captured.'); +} + +main();