Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tests enable register test and windows support #2374

Merged
merged 6 commits into from
Nov 11, 2024
Merged

Tests enable register test and windows support #2374

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f62d02d
test: add vault register check
ermineJose Nov 4, 2024
1ffd45b
test: add windows support to vault CLI
ermineJose Nov 4, 2024
d0ead24
fix: errors related to windows/linux
ermineJose Nov 5, 2024
aa25b3a
fix: ensure workflow stops on error
ermineJose Nov 5, 2024
911d57e
test: add python logic for comparison
ermineJose Nov 6, 2024
bba7bbb
Merge branch 'main' into tests-enable_register_test
ermineJose Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 108 additions & 17 deletions .github/workflows/merge.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,29 +357,40 @@ jobs:
SN_LOG: "v"
timeout-minutes: 2

- name: create local user file
run: echo random > random.txt
env:
SN_LOG: "v"
timeout-minutes: 2

- name: file upload
run: ./target/release/autonomi --log-output-dest=data-dir file upload random.txt
env:
SN_LOG: "v"
timeout-minutes: 2

- name: create a local register
run: ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234
env:
SN_LOG: "v"
timeout-minutes: 2

- name: Estimate cost to create a vault
if: matrix.os != 'windows-latest'
run: |
echo "test-file" > upload-test.txt
./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt
./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234
./target/release/autonomi --log-output-dest=data-dir vault cost
./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt
./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt
run: ./target/release/autonomi --log-output-dest=data-dir vault cost
env:
SN_LOG: "v"
timeout-minutes: 2

- name: create a vault with existing user data as above
if: matrix.os != 'windows-latest'
run: ./target/release/autonomi --log-output-dest=data-dir vault create
env:
SN_LOG: "v"
timeout-minutes: 2

- name: add more files
- name: add more files - linux/macos
if: matrix.os != 'windows-latest'
run: |
set -e
for i in {1..100}; do
dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none
./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin --public
Expand All @@ -390,16 +401,36 @@ jobs:
SN_LOG: "v"
timeout-minutes: 25

- name: add more files - windows
if: matrix.os == 'windows-latest'
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
for ($i = 1; $i -le 100; $i++) {
$fileName = "random_file_$i.bin"
$byteArray = [byte[]]@(0xFF) * (1MB) # Create a 1 MB array filled with 0xFF
[System.IO.File]::WriteAllBytes($fileName, $byteArray)

# Run autonomi commands
./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" --public
./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin"
./target/release/autonomi --log-output-dest=data-dir register create $i "random_file_$i.bin"
}
env:
SN_LOG: "v"
timeout-minutes: 25

- name: sync the vault
if: matrix.os != 'windows-latest'
run: ./target/release/autonomi --log-output-dest=data-dir vault sync
env:
SN_LOG: "v"
timeout-minutes: 2

- name: vault sync validation
if: matrix.os != 'windows-latest'
shell: bash
run: |
set -e
NUM_OF_PUBLIC_FILES=""
NUM_OF_PRIVATE_FILES=""
NUM_OF_REGISTERS=""
Expand All @@ -409,34 +440,93 @@ jobs:

./target/release/autonomi --log-output-dest=data-dir file list 2>&1 > file_list.txt

# ./target/release/autonomi --log-output-dest=data-dir register list | grep archives > register_list.txt
./target/release/autonomi register list | grep register > register_list.txt

NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'`
NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'`
NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'`

# when obtaining registers we get random garbage, this is the only hack that works.
NUM_OF_REGISTERS_first=${NUM_OF_REGISTERS%%[ $'\n']*}
echo "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first"
./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 > vault_data.txt

NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'`
NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'`
# NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'`
NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'`

echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT"
echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT"
# echo "Total Num of local registers is $NUM_OF_REGISTERS and in vault is $NUM_OF_REGISTERS_IN_VAULT"
echo "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT"

rm -rf file_list.txt register_list.txt vault_data.txt

python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT
python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT
# python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT
python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS_first $NUM_OF_REGISTERS_IN_VAULT
echo "vault synced successfully!"
env:
SN_LOG: "v"
timeout-minutes: 15

- name: Set up variables - vault sync - windows
if: matrix.os == 'windows-latest'
shell: pwsh
run: |
RolandSherwin marked this conversation as resolved.
Show resolved Hide resolved
$ErrorActionPreference = "Stop"
./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1
./target/release/autonomi register list > register_list.txt 2>&1
./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1
env:
SN_LOG: "v"
timeout-minutes: 15

- name: Vault sync validation
if: matrix.os == 'windows-latest'
shell: python
run: |
import re
def find_number_before_word(file_name, search_word):
"""
Reads a file and finds the number immediately preceding a specified word in a line.

:param file_name: Name of the file to read.
:param search_word: Word to search for in the file.
:return: The number before the word as an integer, or None if not found.
"""
try:
with open(file_name, 'r') as file:
for line in file:
if search_word in line:
match = re.search(r'(\d+)\s+' + re.escape(search_word), line)
if match:
return int(match.group(1)) # Convert to integer
return None # Return None if no match is found
except FileNotFoundError:
print(f"Error: File '{file_name}' not found.")
return None
NUM_OF_PUBLIC_FILES = find_number_before_word("file_list.txt", "public")
print("NUM_OF_PUBLIC_FILES:", NUM_OF_PUBLIC_FILES)
Comment on lines +507 to +508
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we not directly use rg here instead of using Python?
Assuming the file content is something like the one below, then rg should suffice.

❯ echo "1000 public\n2000 private" | rg '\d+\s+public' | awk '{print $1}'
1000

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not working and python makes it easier to read from the file and make a comparison operation within the one job in windows.

NUM_OF_PRIVATE_FILES = find_number_before_word("file_list.txt", "private")
print("NUM_OF_PRIVATE_FILES:", NUM_OF_PRIVATE_FILES)
NUM_OF_REGISTERS_FILES = find_number_before_word("register_list.txt", "register")
print("NUM_OF_REGISTERS_FILES:", NUM_OF_REGISTERS_FILES)
NUM_OF_PUBLIC_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "public")
print("NUM_OF_PUBLIC_FILES_IN_VAULT:", NUM_OF_PUBLIC_FILES_IN_VAULT)
NUM_OF_PRIVATE_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "private")
print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT)
NUM_OF_REGISTERS_IN_VAULT = find_number_before_word("vault_data.txt", "register")
print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT)

# Assertions
assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal"
assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal"
assert NUM_OF_REGISTERS_FILES == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FILES} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal"
print("Vault synced successfully!")
env:
SN_LOG: "v"
timeout-minutes: 2

- name: load an existing vault from the network
if: matrix.os != 'windows-latest'
run: ./target/release/autonomi --log-output-dest=data-dir vault load
env:
SN_LOG: "v"
Expand All @@ -445,6 +535,7 @@ jobs:
- name: Time profiling for Different files
if: matrix.os != 'windows-latest'
run: |
set -e
# 1 MB
python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))"
# 10 MB
Expand Down
Loading