[indexer] use in memory buffer to store obj changes and update snapsh… #69359

Workflow file for this run

	name: Rust

	on:
	push:
	branches:
	- 'main'
	- 'devnet'
	- 'testnet'
	- 'mainnet'
	- 'releases/sui-*-release'
	pull_request:
	types: [opened, synchronize, reopened, ready_for_review]

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	env:
	CARGO_TERM_COLOR: always
	# Disable incremental compilation.
	#
	# Incremental compilation is useful as part of an edit-build-test-edit cycle,
	# as it lets the compiler avoid recompiling code that hasn't changed. However,
	# on CI, we're not making small edits; we're almost always building the entire
	# project from scratch. Thus, incremental compilation on CI actually
	# introduces additional overhead to support making future builds
	# faster...but no future builds will ever occur in any given CI environment.
	#
	# See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
	# for details.
	CARGO_INCREMENTAL: 0
	# Allow more retries for network requests in cargo (downloading crates) and
	# rustup (installing toolchains). This should help to reduce flaky CI failures
	# from transient network timeouts or other issues.
	CARGO_NET_RETRY: 10
	RUSTUP_MAX_RETRIES: 10
	# Don't emit giant backtraces in the CI logs.
	RUST_BACKTRACE: short
	# RUSTFLAGS: -D warnings
	RUSTDOCFLAGS: -D warnings

	jobs:
	diff:
	runs-on: [ubuntu-latest]
	outputs:
	isRust: ${{ steps.diff.outputs.isRust }}
	isMove: ${{ steps.diff.outputs.isMove }}
	isSolidity: ${{ steps.diff.outputs.isSolidity }}
	isReleaseNotesEligible: ${{ steps.diff.outputs.isReleaseNotesEligible }}
	isMoveAutoFormatter: ${{ steps.diff.outputs.isMoveAutoFormatter }}
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- name: Detect Changes
	uses: './.github/actions/diffs'
	id: diff

	license-check:
	name: license-check
	runs-on: [ubuntu-latest]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths
	- name: Install cargo-hakari, and cache the binary
	uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # [email protected]
	with:
	crate: cargo-hakari
	locked: true
	- run: cargo xlint

	release-notes-description-check:
	name: release-notes-check
	needs: diff
	if: needs.diff.outputs.isReleaseNotesEligible == 'true'
	runs-on: [ubuntu-latest]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # [email protected]
	with:
	python-version: 3.10.10
	- name: Validate PR's release notes
	shell: bash
	run: \|
	python ./scripts/release_notes.py check

	move-auto-formatter-ci-test:
	needs: diff
	if: needs.diff.outputs.isMoveAutoFormatter == 'true'
	runs-on: [ubuntu-latest]
	strategy:
	matrix:
	node-version: [20]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # [email protected]
	with:
	version: 9.1.1
	- name: Setup Node
	uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # [email protected]
	with:
	node-version: '20.x'
	cache: 'pnpm'
	- name: Install dependencies
	working-directory: ./external-crates/move/crates/move-analyzer/prettier-plugin
	run: npm install && npm i web-tree-sitter
	- name: Run npm test
	working-directory: ./external-crates/move/crates/move-analyzer/prettier-plugin
	shell: bash
	run: npm run test

	test:
	needs: diff
	if: needs.diff.outputs.isRust == 'true' \|\| needs.diff.outputs.isSolidity == 'true'
	timeout-minutes: 45
	env:
	# Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var
	# causes #[sim_test] to only run under the deterministic `simtest` job, and not the
	# non-deterministic `test` job.
	SUI_SKIP_SIMTESTS: 1
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os:
	- [ubuntu-ghcloud]
	fail-fast: false
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- name: Set Swap Space
	uses: pierotofy/set-swap-space@master
	with:
	swap-size-gb: 256
	- name: Install Foundry
	run: \|
	curl -L https://foundry.paradigm.xyz \| { cat; echo '$FOUNDRY_BIN_DIR/foundryup'; } \| bash
	echo "$HOME/.config/.foundry/bin" >> $GITHUB_PATH
	- name: Install Foundry Dependencies
	working-directory: bridge/evm
	run: \|
	forge install https://github.com/OpenZeppelin/[email protected] https://github.com/foundry-rs/[email protected] https://github.com/OpenZeppelin/openzeppelin-foundry-upgrades --no-git --no-commit
	- name: cargo test
	run: \|
	cargo nextest run --profile ci
	# Ensure there are no uncommitted changes in the repo after running tests
	- run: scripts/changed-files.sh
	shell: bash

	test-extra:
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	timeout-minutes: 45
	env:
	# Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var
	# causes #[sim_test] to only run under the deterministic `simtest` job, and not the
	# non-deterministic `test` job.
	SUI_SKIP_SIMTESTS: 1
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os:
	- [ubuntu-ghcloud]
	fail-fast: false
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- name: Set Swap Space
	uses: pierotofy/set-swap-space@master
	with:
	swap-size-gb: 256
	- name: benchmark (smoke)
	run: \|
	cargo run --package sui-benchmark --bin stress -- --log-path /tmp/stress.log --num-client-threads 10 --num-server-threads 24 --num-transfer-accounts 2 bench --target-qps 100 --num-workers 10 --transfer-object 50 --shared-counter 50 --run-duration 10s --stress-stat-collection
	- name: doctests
	run: \|
	cargo test --doc
	- name: rustdoc
	run: \|
	cargo doc --workspace --no-deps
	- name: Install cargo-hakari, and cache the binary
	uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # [email protected]
	with:
	crate: cargo-hakari
	locked: true
	- name: Install rustfmt
	run: rustup component add rustfmt
	- name: sui-execution
	run: \|
	./scripts/execution_layer.py generate-lib
	# Ensure there are no uncommitted changes in the repo after running tests
	- run: scripts/changed-files.sh
	shell: bash

	windows-build:
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	timeout-minutes: 45
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os:
	- [windows-ghcloud]
	fail-fast: false
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # [email protected]
	# this avoids rate-limiting
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	- name: Install postgres (Windows)
	shell: bash
	run: \|
	choco install postgresql12 --force --params '/Password:root'
	echo "C:\Program Files\PostgreSQL\12\bin" >> $GITHUB_PATH
	echo "C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_PATH
	echo "PQ_LIB_DIR=C:\Program Files\PostgreSQL\12\lib" >> $GITHUB_ENV
	echo "PG_DATABASE_URL=postgres://postgres:root@localhost/" >> $GITHUB_ENV
	echo "PG_EXAMPLE_DATABASE_URL=postgres://postgres:root@localhost/diesel_example" >> $GITHUB_ENV

	- name: Install MySql (Windows)
	shell: bash
	run: \|
	choco install mysql --force --params '/Password:root'
	echo "MYSQLCLIENT_LIB_DIR=C:\Program Files\MySQL\MySQL Server 8.0\lib" >> $GITHUB_ENV

	- name: cargo build
	shell: bash
	run: \|
	cargo build --all-features

	simtest:
	needs: diff
	if: needs.diff.outputs.isRust == 'true' \|\| needs.diff.outputs.isSolidity == 'true'
	timeout-minutes: 45
	runs-on: [ubuntu-ghcloud]
	env:
	MSIM_WATCHDOG_TIMEOUT_MS: 60000
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- name: Set Swap Space
	uses: pierotofy/set-swap-space@master
	with:
	swap-size-gb: 256
	- name: cargo simtest
	run: \|
	scripts/simtest/cargo-simtest simtest
	- name: check new tests for flakiness
	run: \|
	scripts/simtest/stress-new-tests.sh

	# This job ensures that Move unit tests are run if there are changes
	# to Move code but not Rust code (If there are Rust changes, they
	# will be run as part of a larger test suite).
	move-test:
	needs: diff
	if: needs.diff.outputs.isRust == 'false' && needs.diff.outputs.isMove == 'true'
	timeout-minutes: 10
	runs-on: [ubuntu-ghcloud]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- name: Run move tests
	run: \|
	cargo nextest run -p sui-framework-tests -- unit_tests::

	# # Disabled
	# rosetta-validation:
	# needs: diff
	# if: needs.diff.outputs.isRust == 'true'
	# timeout-minutes: 45
	# runs-on: [ubuntu-ghcloud]
	# steps:
	# - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1

	# - name: Setup environment
	# run: .github/scripts/rosetta/setup.sh
	# shell: bash

	# - name: Start local Sui network
	# run: \|
	# sui start --no-full-node &
	# shell: bash

	# - name: Start Rosetta servers
	# run: .github/scripts/rosetta/start_rosetta.sh
	# shell: bash

	# - name: Sleep for 20 seconds
	# run: sleep 20s
	# shell: bash

	# - name: Run check:construction test
	# run: \|
	# ./bin/rosetta-cli --configuration-file rosetta_cli.json check:construction
	# shell: bash

	# - name: Run check:data test
	# run: \|
	# ./bin/rosetta-cli --configuration-file rosetta_cli.json check:data
	# shell: bash

	# This is a no-op job that allows the resulting action names to line up when
	# there are no rust changes in a given PR/commit. This ensures that we can
	# continue to block on the rust tests passing in the case of rust changes and
	# otherwise not block pushes to main.
	# Note that this no-op job is only when there is a strategy matrix.
	# Other required jobs without strategy matrix do not need a no-op counterpart.
	test-notrust:
	name: test
	needs: diff
	if: needs.diff.outputs.isRust == 'false'
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os:
	- [ubuntu-ghcloud]
	- [windows-ghcloud]
	fail-fast: false
	steps:
	- run: 'echo "No build required" '

	# This is a no-op job that allows the resulting action names to line up when
	# there are no rust changes in a given PR/commit. This ensures that we can
	# continue to block on the rust tests passing in the case of rust changes and
	# otherwise not block pushes to main.
	# Note that this no-op job is needed when there is a strategy matrix.
	# Other required jobs without strategy matrix do not need a no-op counterpart.
	test-extra-notrust:
	name: test-extra
	needs: diff
	if: needs.diff.outputs.isRust == 'false'
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os:
	- [ubuntu-ghcloud]
	- [windows-ghcloud]
	fail-fast: false
	steps:
	- run: 'echo "No build required" '

	clippy:
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	runs-on: [ubuntu-ghcloud]
	steps:
	- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # [email protected]
	# this avoids rate-limiting
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- run: rustup component add clippy
	# TODO(bradh): debug and re-enable this; the caching is breaking the clippy build
	# Enable caching of the 'librocksdb-sys' crate by additionally caching the
	# 'librocksdb-sys' src directory which is managed by cargo
	# - uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths
	# with:
	# path: ~/.cargo/registry/src/*/librocksdb-sys-

	# See '.cargo/config' for list of enabled/disappled clippy lints
	- name: cargo clippy
	run: cargo xclippy -D warnings

	- name: cargo mysql-clippy
	run: cargo mysql-clippy -D warnings

	rustfmt:
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	runs-on: [ubuntu-latest]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- run: rustup component add rustfmt
	- name: rustfmt
	uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # [email protected]
	with:
	command: fmt
	args: --check

	cargo-deny:
	name: cargo-deny (advisories, licenses, bans, ...)
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	runs-on: [ubuntu-latest]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: mystenlabs/cargo-deny-action@main

	sui-excution-cut:
	name: cutting a new execution layer
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	runs-on: [ubuntu-ghcloud]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- name: Install cargo-hakari, and cache the binary
	uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # [email protected]
	with:
	crate: cargo-hakari
	locked: true
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- name: Make cut
	run: ./scripts/execution_layer.py cut for_ci_test
	- name: Check execution builds
	run: cargo build -p sui-execution

	#indexer:
	# name: indexer
	# needs: diff
	# if: needs.diff.outputs.isRust == 'true'
	# timeout-minutes: 45
	# runs-on: [ubuntu-ghcloud]
	# services:
	# postgres:
	# image: postgres
	# env:
	# POSTGRES_PASSWORD: postgrespw
	# options: >-
	# --health-cmd pg_isready
	# --health-interval 10s
	# --health-timeout 5s
	# --health-retries 5
	# ports:
	# - 5432:5432
	# steps:
	# - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	# - name: integration-test-with-postgres
	# #We only have 1 instance of postgres DB, can only run test in single thread
	# run: \|
	# cargo test --package sui-indexer --test integration_tests --features pg_integration -- --test-threads=1
	# env:
	# POSTGRES_HOST: localhost
	# POSTGRES_PORT: 5432

	graphql-rpc:
	name: graphql-rpc
	needs: diff
	if: needs.diff.outputs.isRust == 'true'
	timeout-minutes: 45
	runs-on: [ubuntu-ghcloud]
	services:
	postgres:
	image: postgres
	env:
	POSTGRES_PASSWORD: postgrespw
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--name postgres_container
	ports:
	- 5432:5432
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- uses: taiki-e/install-action@nextest
	- name: Setup db
	run: \|
	PGPASSWORD=$POSTGRES_PASSWORD psql -h localhost -U $POSTGRES_USER -c 'CREATE DATABASE sui_indexer;' -c 'ALTER SYSTEM SET max_connections = 500;'
	env:
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgrespw
	- run: docker restart --time 0 postgres_container
	- run: sleep 5
	- name: tests-requiring-postgres
	run: \|
	cargo nextest run --test-threads 1 --package sui-graphql-rpc --test e2e_tests --test examples_validation_tests --features pg_integration
	cargo nextest run --test-threads 1 --package sui-graphql-rpc --lib --features pg_integration -- test_query_cost
	cargo nextest run --test-threads 4 --package sui-graphql-e2e-tests --features pg_integration
	cargo nextest run --test-threads 1 --package sui-cluster-test --test local_cluster_test --features pg_integration
	cargo nextest run --test-threads 1 --package sui-indexer --test ingestion_tests --features pg_integration

	env:
	POSTGRES_HOST: localhost
	POSTGRES_PORT: 5432

	bridge-evm:
	name: bridge-evm-tests
	needs: diff
	if: needs.diff.outputs.isSolidity == 'true'
	runs-on: [ubuntu-latest]
	steps:
	- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
	- name: Install Foundry
	run: \|
	curl -L https://foundry.paradigm.xyz \| { cat; echo '$FOUNDRY_BIN_DIR/foundryup'; } \| bash
	echo "$HOME/.config/.foundry/bin" >> $GITHUB_PATH
	- name: Install Foundry Dependencies
	working-directory: bridge/evm
	run: \|
	forge install https://github.com/OpenZeppelin/[email protected] https://github.com/foundry-rs/[email protected] https://github.com/OpenZeppelin/openzeppelin-foundry-upgrades --no-git --no-commit
	- name: Check Bridge EVM Unit Tests
	shell: bash
	working-directory: bridge/evm
	env:
	INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
	run: \|
	forge clean
	forge test --ffi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[indexer] use in memory buffer to store obj changes and update snapsh… #69359

Workflow file

[indexer] use in memory buffer to store obj changes and update snapsh… #69359

Jobs

Run details

Workflow file for this run