Use BOSS indexing in DBGSuccinct + make RowDiff independent

.github/workflows/main.yml

@@ -7,8 +7,6 @@ on:
     tags:
       - 'v*'
   pull_request:
-    branches:
-      - master
 
 env:
   REGISTRY: ghcr.io
@@ -127,9 +125,9 @@ jobs:
       run: mv metagraph/build/metagraph_${{ matrix.alphabet }} metagraph/build/metagraph_${{ matrix.alphabet }}_noAVX
     - name: upload static binary
       if: ${{ matrix.build_static == 'ON' && matrix.compiler == 'g++-11' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: metagraph_${{ matrix.alphabet }}_linux_x86
+        name: metagraph_${{ matrix.alphabet }}${{ matrix.with_avx == 'OFF' && '_noAVX' || '' }}_linux_x86
         path: metagraph/build/metagraph_${{ matrix.alphabet }}${{ matrix.with_avx == 'OFF' && '_noAVX' || '' }}
 
     - name: run unit tests
@@ -213,7 +211,7 @@ jobs:
               python-version: 3.8
 
         - name: fetch static binary
-          uses: actions/download-artifact@v2
+          uses: actions/download-artifact@v4
           with:
               path: artifacts
 
@@ -285,7 +283,7 @@ jobs:
         run: git submodule update --init --recursive
 
       - name: fetch static binary
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 

metagraph/integration_tests/base.py

@@ -37,10 +37,19 @@ def setUpClass(cls):
     def _get_stats(graph_path):
         stats_command = METAGRAPH + ' stats ' + graph_path + ' --mmap'
         res = subprocess.run(stats_command.split(), stdout=PIPE, stderr=PIPE)
-        assert(res.returncode == 0)
+        if res.returncode != 0:
+            raise AssertionError(f"Command '{stats_command}' failed with return code {res.returncode} and error: {res.stderr.decode()}")
         stats_command = METAGRAPH + ' stats ' + graph_path + MMAP_FLAG
         res = subprocess.run(stats_command.split(), stdout=PIPE, stderr=PIPE)
-        return res
+        parsed = dict()
+        parsed['returncode'] = res.returncode
+        res = res.stdout.decode().split('\n')[2:]
+        for line in res:
+            if ': ' in line:
+                x, y = map(str.strip, line.split(':', 1))
+                assert(x not in parsed or parsed[x] == y)
+                parsed[x] = y
+        return parsed
 
     @staticmethod
     def _build_graph(input, output, k, repr, mode='basic', extra_params=''):

metagraph/integration_tests/test_align.py

@@ -35,11 +35,10 @@ def test_simple_align_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --align-only-forwards -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -68,11 +67,10 @@ def test_simple_align_map_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers {reads}'.format(
             exe=METAGRAPH,
@@ -99,11 +97,10 @@ def test_simple_align_map_all_graphs_subk(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers --align-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -134,11 +131,10 @@ def test_simple_align_map_canonical_all_graphs(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers {reads}'.format(
             exe=METAGRAPH,
@@ -165,11 +161,10 @@ def test_simple_align_json_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --align-only-forwards -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -189,11 +184,10 @@ def test_simple_align_fwd_rev_comp_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -222,11 +216,10 @@ def test_simple_align_canonical_all_graphs(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -256,11 +249,10 @@ def test_simple_align_canonical_subk_succinct(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 --align-min-seed-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -286,11 +278,10 @@ def test_simple_align_primary_all_graphs(self, representation):
                           k=11, repr=representation, mode='primary',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16391', params_str[1])
-        self.assertEqual('mode: primary', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16391', params['nodes (k)'])
+        self.assertEqual('primary', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -320,11 +311,10 @@ def test_simple_align_primary_subk_succinct(self, representation):
                           k=11, repr=representation, mode='primary',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16391', params_str[1])
-        self.assertEqual('mode: primary', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16391', params['nodes (k)'])
+        self.assertEqual('primary', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 --align-min-seed-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -349,11 +339,10 @@ def test_simple_align_fwd_rev_comp_json_all_graphs(self, representation):
                           output=self.tempdir.name + '/genome.MT',
                           k=11, repr=representation)
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16461', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16461', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --json -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -375,11 +364,10 @@ def test_simple_align_edit_distance_all_graphs(self, representation):
                           output=self.tempdir.name + '/genome.MT',
                           k=11, repr=representation)
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16461', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16461', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --json --align-edit-distance -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,