Fix monorepo build pipeline (#449)

.gitignore

@@ -1,5 +1,5 @@
 **/dist
-**/*__GENERATED*
+**/*__GENERATED__*
 
 # sed backup files
 *.*-e

README.md

@@ -14,13 +14,26 @@ This is currently the only way to use `@lavamoat/allow-scripts` in monorepos.
 
 ### Building
 
-For local development, you should run `yarn build:clean` in the project root directory.
-This will always build the packages in the correct order.
+Run `yarn build` to build all packages in correct order.
+If you encounter any errors, try `yarn build:clean`, and if that fails, check the TypeScript configuration (see below).
 
-You can also run `yarn build` in a workspace, although you have to ensure that the projects are built in the correct order.
+You can also run `yarn build` in a specific package / workspace, although you have to ensure that its dependencies have been built.
 
 Repository-wide watching is currently not possible due to the build processes of some packages.
 
+#### Configuring TypeScript
+
+The TypeScript configuration of this monorepo is brittle, and requires manual maintenance.
+It uses TypeScript [project references](https://www.typescriptlang.org/docs/handbook/project-references.html) and `composite` sub-projects (i.e. monorepo package).
+In short, the [root `tsconfig.json`](./tsconfig.json) must contain an empty `files` array, and `references` to each interdependent project with a `tsconfig.json` in its root directory.
+Meanwhile, every sub-project must explicitly declare the relative paths to its local dependencies via its `references` array.
+
+If building from the monorepo root suddenly starts to fail, check if the errors are referring to monorepo packages, and verify that their `tsconfig.json` files are configured correctly.
+
+Some packages do not require a `tsconfig.json` file.
+These packages must be explicitly ignored in the [TypeScript config lint script](./scripts/verify-tsconfig.mjs).
+If a package is neither referenced nor ignored, linting will fail.
+
 ### Testing and Linting
 
 Run `yarn test` and `yarn lint` in the project root directory, or in a workspace.

package.json

@@ -17,11 +17,14 @@
     "lint:eslint": "eslint . --cache --ext js,ts",
     "lint:misc": "prettier '**/*.json' '**/*.md' '!**/CHANGELOG.md' '**/*.yml' --ignore-path .gitignore",
     "lint:changelogs": "yarn workspaces run lint:changelog",
-    "lint": "yarn lint:eslint && yarn lint:misc --check",
-    "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
-    "build:tsc": "tsc --build --force tsconfig.json",
-    "build": "yarn workspaces run build:pre-tsc && yarn build:tsc && yarn workspaces run build:post-tsc",
+    "lint:tsconfig": "node scripts/verify-tsconfig.mjs",
+    "lint": "yarn lint:tsconfig && yarn lint:eslint && yarn lint:misc --check",
+    "lint:fix": "yarn lint:tsconfig && yarn lint:eslint --fix && yarn lint:misc --write",
+    "build": "yarn build:pre-tsc && yarn build:tsc && yarn build:post-tsc",
     "build:clean": "yarn clean && yarn build",
+    "build:tsc": "tsc --build",
+    "build:pre-tsc": "yarn workspaces run build:pre-tsc ",
+    "build:post-tsc": "yarn workspaces run build:post-tsc",
     "clean": "yarn workspaces run clean",
     "test": "yarn workspaces run test",
     "test:ci": "yarn workspaces run test:ci"

packages/cli/package.json

@@ -26,7 +26,7 @@
     "build:post-tsc": "yarn build:chmod && yarn build:readme",
     "build:clean": "yarn clean && yarn build",
     "build:watch": "tsc-watch --onSuccess 'yarn build:chmod'",
-    "clean": "rimraf dist/* src/**/*__GENERATED__*",
+    "clean": "rimraf *.tsbuildinfo dist/* src/**/*__GENERATED__*",
     "test": "yarn build:init-template && jest",
     "posttest": "jest-it-up",
     "test:watch": "yarn test --watch",

packages/cli/tsconfig.json

@@ -1,13 +1,11 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": true,
-    "forceConsistentCasingInFileNames": true,
     "inlineSources": true,
     "lib": ["DOM", "ES2020"],
-    "outDir": "dist",
-    "rootDir": "src",
+    "outDir": "./dist",
     "resolveJsonModule": true,
+    "rootDir": "./src",
     "skipLibCheck": false,
     "typeRoots": [
       "../../node_modules/@types",
@@ -16,5 +14,8 @@
     ]
   },
   "include": ["./src/**/*.ts", "./src/**/*.json"],
-  "exclude": ["./src/**/*.test.ts"]
+  "references": [
+    { "path": "../controllers" },
+    { "path": "../plugin-browserify" }
+  ]
 }

packages/controllers/jest.config.js

@@ -8,7 +8,7 @@ module.exports = {
   coveragePathIgnorePatterns: ['/node_modules/', '/mocks/', '/test/'],
   coverageThreshold: {
     global: {
-      branches: 66.67,
+      branches: 64.44,
       functions: 82.11,
       lines: 82.17,
       statements: 82.23,

packages/controllers/package.json

@@ -23,7 +23,7 @@
     "build:ajv": "./scripts/build-ajv.sh",
     "build": "yarn build:pre-tsc && yarn build:tsc",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "lint:eslint": "eslint . --cache --ext js,ts",
     "lint:misc": "prettier '**/*.json' '**/*.md' '!CHANGELOG.md' --ignore-path ../../.gitignore",
     "lint": "yarn lint:eslint && yarn lint:misc --check",

packages/controllers/tsconfig.json

@@ -1,15 +1,13 @@
 {
   "extends": "../../tsconfig.packages.json",
-  "references": [{ "path": "../execution-environments" }],
   "compilerOptions": {
     "allowJs": true,
-    "composite": true,
     "lib": ["DOM", "ES2020"],
-    "outDir": "dist",
+    "outDir": "./dist",
     "resolveJsonModule": true,
-    "rootDir": "src",
+    "rootDir": "./src",
     "typeRoots": ["../../node_modules/@types", "./node_modules/@types"]
   },
   "include": ["./src", "./src/snaps/json-schemas"],
-  "exclude": ["**/*.test.ts"]
+  "references": [{ "path": "../execution-environments" }]
 }

packages/examples/examples/notifications/package.json

@@ -18,7 +18,7 @@
     "build:website": "node ./scripts/build-website.js",
     "build": "mm-snap build",
     "serve": "mm-snap serve",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "test": "echo 'TODO'",
     "lint:eslint": "eslint . --cache --ext js,ts",
     "lint:misc": "prettier '**/*.json' '**/*.md' '!CHANGELOG.md' --ignore-path .gitignore",

packages/execution-environments/jest.config.js

@@ -6,10 +6,10 @@ module.exports = {
   coverageReporters: ['clover', 'json', 'lcov', 'text', 'json-summary'],
   coverageThreshold: {
     global: {
-      branches: 29.82,
+      branches: 34.69,
       functions: 38.89,
-      lines: 34.87,
-      statements: 35.47,
+      lines: 34.62,
+      statements: 34.98,
     },
   },
   moduleFileExtensions: ['js', 'json', 'jsx', 'ts', 'tsx', 'node'],

packages/execution-environments/package.json

@@ -23,7 +23,7 @@
     "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
     "lint:changelog": "yarn auto-changelog validate",
     "lint": "yarn lint:eslint && yarn lint:misc --check",
-    "clean": "rimraf dist src/__GENERATED__/",
+    "clean": "rimraf *.tsbuildinfo dist/* src/__GENERATED__/",
     "build:clean": "yarn clean && yarn build",
     "build:pre-tsc": "yarn build:typings",
     "build:post-tsc": "webpack --mode production && concat -o ./dist/webpack/webworker/bundle.js ./dist/webpack/webworker/lockdown.umd.min.js ./dist/webpack/webworker/bundle.js",

packages/execution-environments/tsconfig.json

@@ -1,13 +1,10 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": true,
-    "outDir": "dist",
-    "rootDir": "src",
-    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
     "typeRoots": ["./node_modules/@types", "../../node_modules/@types"],
     "resolveJsonModule": true
   },
-  "include": ["./src", "./src/openrpc.json"],
-  "exclude": ["**/*.test.ts"]
+  "include": ["./src", "./src/openrpc.json"]
 }

packages/plugin-browserify/package.json

@@ -21,12 +21,12 @@
     "lint": "yarn lint:eslint && yarn lint:misc --check",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
     "lint:changelog": "yarn auto-changelog validate",
-    "build:tsc": "tsc --project tsconfig.build.json",
+    "build:tsc": "tsc --project tsconfig.local.json",
     "build": "yarn build:tsc",
     "build:pre-tsc": "echo 'N/A'",
     "build:post-tsc": "echo 'N/A'",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "publish": "../../scripts/publish-package.sh"
   },
   "dependencies": {

packages/plugin-browserify/tsconfig.json

@@ -1,10 +1,10 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": false,
     "outDir": "./dist",
     "rootDir": "./src",
     "typeRoots": ["./node_modules/@types"]
   },
-  "include": ["./src"]
+  "include": ["./src"],
+  "references": [{ "path": "../utils" }]
 }

packages/plugin-browserify/tsconfig.local.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "composite": false
+  }
+}

packages/plugin-rollup/package.json

@@ -22,12 +22,12 @@
     "lint": "yarn lint:eslint && yarn lint:misc --check",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
     "lint:changelog": "yarn auto-changelog validate",
-    "build:tsc": "tsc --project tsconfig.build.json",
+    "build:tsc": "tsc --project tsconfig.local.json",
     "build": "yarn build:tsc",
     "build:pre-tsc": "echo 'N/A'",
     "build:post-tsc": "echo 'N/A'",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "publish": "../../scripts/publish-package.sh"
   },
   "dependencies": {

packages/plugin-rollup/tsconfig.json

@@ -1,10 +1,10 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": false,
     "outDir": "./dist",
     "rootDir": "./src",
     "typeRoots": ["./node_modules/@types"]
   },
-  "include": ["./src"]
+  "include": ["./src"],
+  "references": [{ "path": "../utils" }]
 }

packages/plugin-rollup/tsconfig.local.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "composite": false
+  }
+}

packages/plugin-webpack/package.json

@@ -22,12 +22,12 @@
     "lint": "yarn lint:eslint && yarn lint:misc --check",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
     "lint:changelog": "yarn auto-changelog validate",
-    "build:tsc": "tsc --project tsconfig.build.json",
+    "build:tsc": "tsc --project tsconfig.local.json",
     "build": "yarn build:tsc",
     "build:pre-tsc": "echo 'N/A'",
     "build:post-tsc": "echo 'N/A'",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "publish": "../../scripts/publish-package.sh"
   },
   "dependencies": {

packages/plugin-webpack/tsconfig.json

@@ -1,10 +1,10 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": false,
     "outDir": "./dist",
     "rootDir": "./src",
     "typeRoots": ["./node_modules/@types"]
   },
-  "include": ["./src"]
+  "include": ["./src"],
+  "references": [{ "path": "../utils" }]
 }

packages/plugin-webpack/tsconfig.local.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "composite": false
+  }
+}

packages/rpc-methods/package.json

@@ -23,7 +23,7 @@
     "build:post-tsc": "echo 'N/A'",
     "build": "tsc --project tsconfig.local.json",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "publish": "../../scripts/publish-package.sh"
   },
   "dependencies": {

packages/rpc-methods/tsconfig.json

@@ -1,11 +1,10 @@
 {
   "extends": "../../tsconfig.packages.json",
-  "references": [{ "path": "../controllers" }],
   "compilerOptions": {
-    "composite": true,
-    "outDir": "dist",
-    "rootDir": "src",
+    "outDir": "./dist",
+    "rootDir": "./src",
     "typeRoots": ["../../node_modules/@types", "./types"]
   },
-  "include": ["./src"]
+  "include": ["./src"],
+  "references": [{ "path": "../controllers" }]
 }

packages/utils/package.json

@@ -18,12 +18,12 @@
     "lint": "yarn lint:eslint && yarn lint:misc --check",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:misc --write",
     "lint:changelog": "yarn auto-changelog validate",
-    "build:tsc": "tsc --project tsconfig.build.json",
+    "build:tsc": "tsc --project tsconfig.local.json",
     "build": "yarn build:tsc",
     "build:pre-tsc": "echo 'N/A'",
     "build:post-tsc": "echo 'N/A'",
     "build:clean": "yarn clean && yarn build",
-    "clean": "rimraf dist/*",
+    "clean": "rimraf *.tsbuildinfo dist/*",
     "publish": "../../scripts/publish-package.sh"
   },
   "dependencies": {

packages/utils/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "../../tsconfig.packages.json",
   "compilerOptions": {
-    "composite": false,
     "outDir": "./dist",
     "rootDir": "./src",
     "typeRoots": ["./node_modules/@types"]

packages/utils/tsconfig.local.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "composite": false
+  }
+}

scripts/verify-tsconfig.mjs

@@ -0,0 +1,28 @@
+import { promises as fs } from 'fs';
+import pathUtils from 'path';
+import { fileURLToPath } from 'url';
+
+const cwd = pathUtils.dirname(fileURLToPath(import.meta.url))
+
+// These are the packages we expect to _not_ be referenced in the root tsconfig.
+const IGNORE_LIST = new Set(['examples', 'types']);
+
+// Get reference paths from root tsconfig.json
+const rootTsconfig = JSON.parse(await fs.readFile('./tsconfig.json', { encoding: 'utf8'}));
+const rootTsconfigReferences = new Set(rootTsconfig.references.map(
+  ({ path }) => path.split('/').pop()
+))
+
+// Get the names of all directories in the packages directory
+const packagesPath = pathUtils.resolve(cwd, '../packages');
+const packageDirNames = (await fs.readdir(packagesPath, { withFileTypes: true }))
+  .filter((dirent) => dirent.isDirectory())
+  .map((dirent) => dirent.name);
+
+// Any unreferenced package dirs must either be referenced or ignored
+const unreferencedPackageDirs = packageDirNames.filter((name) => !rootTsconfigReferences.has(name) && !IGNORE_LIST.has(name))
+if (unreferencedPackageDirs.length > 0) {
+  throw new Error(`Found unreferenced package directories not in ignore list:\n\n\t${
+    unreferencedPackageDirs.join('\n\t')
+  }\n\nEither reference or ignore the packages to continue.`)
+}