Merge pull request #2 from DarkEnergySurvey/addingMigrationScript

Added chown capabilities to migration

bin/chown.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+chown -R svcdesdatabot:des_dm $1

bin/migrate.py

@@ -16,12 +16,13 @@
 from filemgmt import fmutils
 from filemgmt import migrate_utils as mu
 
+
 def parse_cmd_line(argv):
     """ Parse command line arguments
 
         Parameters
         ----------
-        args : command line arguments
+        argv : command line arguments
 
         Returns
         -------
@@ -65,12 +66,11 @@ def parse_cmd_line(argv):
     parser.add_argument('--pfwid', action='store', help='pfw attempt id to search for')
     parser.add_argument('--silent', action='store_true', help='Run with minimal printing, only print ERROR or OK')
     parser.add_argument('--tag', action='store', help='Compare all data from a specific tag (this can take a long time)')
-    parser.add_argument('--dbh', action='store', help=argparse.SUPPRESS) # used internally
+    parser.add_argument('--dbh', action='store', help=argparse.SUPPRESS)  # used internally
     parser.add_argument('--log', action='store', help='Log file to write to, default is to write to sdtout')
     parser.add_argument('--parallel', action='store', help='Specify the parallelization of the migration, e.g. 3 would spread the work across 3 subprocesses.', type=int, default=1)
     parser.add_argument('--raw', action='store', default=None, help='Migrate RAW files')
-    parser.add_argument('--user', action='store', default=None, help='User to change ownership to. Default is no chnage.')
-    parser.add_argument('--group', action='store', default=None, help='Group to change ownership to. Default is no change.')
+    parser.add_argument('--chown', action='store_true', default=False, help='User to change ownership to. Default is no chnage.')
     cargs = parser.parse_args(argv)
     if cargs.script:
         cargs.verbose = False
@@ -196,5 +196,6 @@ def interrupt(x, y):
         sys.stdout = stdp.close()
     sys.exit(0)
 
+
 if __name__ == "__main__":
     main()

python/filemgmt/fmutils.py

@@ -443,8 +443,7 @@ def __init__(self, win, args, pfwids, event, dirs=[], que=None):
         self.attnum = check_arg(args, 'attnum')
         self.md5sum = check_arg(args, 'md5sum')
         self.raw = check_arg(args, 'raw')
-        self.user = check_arg(args, 'user')
-        self.group = check_arg(args, 'group')
+        self.chown = check_arg(args, 'chown')
 
         self.dirs = dirs
         self.verbose = args.verbose
@@ -466,7 +465,7 @@ def __init__(self, win, args, pfwids, event, dirs=[], que=None):
         self.db_duplicates = None
         self.files_from_disk = None
         self.duplicates = None
-        self.comparison_info ={}
+        self.comparison_info = {}
 
     def reset(self):
         self.dbh.close()
@@ -479,7 +478,7 @@ def reset(self):
         self.db_duplicates = None
         self.files_from_disk = None
         self.duplicates = None
-        self.comparison_info ={}
+        self.comparison_info = {}
         self._reset()
 
     def _reset(self):

python/filemgmt/migrate_utils.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 import stat
 import time
+import subprocess
 
 from despymisc import miscutils
 from filemgmt import fmutils
@@ -60,19 +61,6 @@ def rollback(self, newpath=None):
     def migrate(self):
         """ Function to copy files from one archive section to another.
 
-            Parameters
-            ----------
-            files_from_db: dict
-                Dictionary of files and descriptors to copy
-
-            current: str
-                The current root path of the files
-
-            destination: str
-                The destination path for the files
-
-            archive_root: str
-                The archive root path
         """
         self.update(f"Copying {self.count} files for {self.relpath}...")
         self.iteration = 0
@@ -96,13 +84,6 @@ def migrate(self):
             try:
                 shutil.copy2(os.path.join(self.archive_root, items['path'], fname), os.path.join(self.archive_root, dst, fname))
                 os.chmod(os.path.join(self.archive_root, dst, fname), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
-                if self.user is not None and self.group is not None:
-                    shutil.chown(os.path.join(self.archive_root, dst, fname), self.user, self.group)
-                if self.user is not None or self.group is not None:
-                    if self.group is not None:
-                        shutil.chown(os.path.join(self.archive_root, dst, fname), group=self.group)
-                    else:
-                        shutil.chown(os.path.join(self.archive_root, dst, fname), self.user)
                 self.copied_files.append(os.path.join(self.archive_root, dst, fname))
             except Exception as ex:
                 self.update(f"Error copying file from {os.path.join(self.archive_root, items['path'], fname)} to {os.path.join(self.archive_root, dst, fname)}", True)
@@ -123,10 +104,6 @@ def migrate(self):
     def do_task(self):
         """ Method to migrate the data
 
-            Parameters
-            ----------
-            args : list of command line arguments
-
             Returns
             -------
             the result
@@ -173,16 +150,26 @@ def do_task(self):
             return 1
         self.update("Updating database...")
         try:
+            curs = self.dbh.cursor()
             if self.results['comp'] :
                 upsql = "update file_archive_info set path=:pth where filename=:fn and compression=:comp"
-                curs = self.dbh.cursor()
                 curs.executemany(upsql, self.results['comp'])
             if self.results['null'] :
                 upsql = "update file_archive_info set path=:pth where filename=:fn and compression is NULL"
-                curs = self.dbh.cursor()
                 curs.executemany(upsql, self.results['null'])
             if self.pfwid:
                 curs.execute(f"update pfw_attempt set archive_path='{newpath}' where id={self.pfwid}")
+                if self.chown:
+                    subp = subprocess.Popen(['sudo', f"{os.environ['FILEMGMT_DIR']}/bin/chown.sh", newarchpath])
+                    while subp.poll() is None:
+                        try:
+                            subp.wait(100)
+                        except subprocess.TimeoutExpired:
+                            self.update("chown process is taking longer than expected.")
+                    retc = subp.poll()
+                    if retc != 0:
+                        self.update(f"chown failed for {newpath}, you may need to run it manually.")
+
         except:
             self.update("Error updating the database entries, rolling back any DB changes.", True)
             time.sleep(2)