fix merge

cern-it-sdc-id · Sep 17, 2019 · 3d277ee · 3d277ee
2 parents 13ff0cb + abfdc76
commit 3d277ee
Show file tree

Hide file tree

Showing 11 changed files with 158 additions and 78 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,7 +1,8 @@
-2019-08-04 Petr Vokac <[email protected]>
-    * fix deprecation warnings
-2019-07-14 Petr Vokac <[email protected]>
-    * LCGDM-2817 allow to configure epsv_match
+2019-09-05 Petr Vokac <[email protected]>
+    * simpify StAR accounting
+    * configure SSR via CGI
+    * fix DB names conf
+    * fix deprecations
 2019-02-26 Andrea Manzi  <[email protected]>
     * fix conf for xrootd tpc delegation
     * raise dome limits

diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ It's the base puppet module for the configuration of Data Management components
 ASL 2.0
 
 ### Contact
-Andrea Manzi <andrea.manzi@cern.ch>
+DPM Devel <dpm-devel-cern@cern.ch>
 
 ## Support
-Tickets and issues at our [cern-it-sdc-id site](https://github.com/cern-it-sdc-id)
+Tickets and issues at our [Jira](https://its.cern.ch/jira/projects/LCGDM)
diff --git a/manifests/accounting.pp b/manifests/accounting.pp
@@ -1,71 +1,114 @@
+# EGI StaR accounting - https://wiki.egi.eu/wiki/APEL/Storage
+# For publishing data in the EGI it is necessary to create
+# eu.egi.storage.accounting service for DPM headnode in GOCDB
+# (https://goc.egi.eu) with "Host DN" set to certificate subject
+# Usage:
+#   class { '::dmlite::accounting':
+#     site_name => 'praguelcg2',
+#   }
 class dmlite::accounting (
-  $cron_interval = hiera('dmlite::accounting::cron_interval','weekly'),
+  $enabled = hiera('dmlite::accounting::enabled',true),
+  $cron_interval = hiera('dmlite::accounting::cron_interval','daily'),
   $bdii_url = hiera('dmlite::accounting::bdii_url','ldap://lcg-bdii.cern.ch:2170'),
   $broker_network = hiera('dmlite::accounting::broker_network','PROD'),
   $broker_host = hiera('dmlite::accounting::broker_host',''),
   $broker_port = hiera('dmlite::accounting::broker_port',''),
-  $use_ssl = hiera('dmlite::accounting::use_ssl',false),
+  $use_ssl = hiera('dmlite::accounting::use_ssl',true),
   $certificate = hiera('dmlite::accounting::certificate','/etc/grid-security/dpmmgr/dpmcert.pem'),
   $key = hiera('dmlite::accounting::key','/etc/grid-security/dpmmgr/dpmkey.pem'),
   $capath = hiera('dmlite::accounting::capath','/etc/grid-security/certificates'),
 
   $server_cert=  hiera('dmlite::accounting::server_cert',''),
-  $messaging_destination = hiera('dmlite::accounting::server_cert','/queue/global.accounting.test.storage.central'),
+  $messaging_destination = hiera('dmlite::accounting::server_cert','/queue/global.accounting.storage.central'),
   $messaging_path = hiera('dmlite::accounting::messaging_path','/var/spool/apel/outgoing'),
-
-
+
   $site_name = hiera('dmlite::accounting::site_name',''),
-  $nsconfig =  hiera('dmlite::accounting::nsconfig','/usr/etc/NSCONFIG'), 
+
+  $nsconfig = hiera('dmlite::accounting::nsconfig','/usr/etc/NSCONFIG'),
+
+  $dbhost = hiera('dmlite::accounting::dbhost','localhost'),
+  $dbuser = hiera('dmlite::accounting::dbuser',''),
+  $dbpwd = hiera('dmlite::accounting::dbpwd',''),
+  $nsdbname = hiera('dmlite::accounting::nsdbname','cns_db'),
+  $dpmdbname = hiera('dmlite::accounting::dpmdbname','dpm_db'),
 
   $log_file =  hiera('dmlite::accounting::log_file','/var/log/apel/ssmsend.log'),
   $log_level = hiera('dmlite::accounting::log_level', 'INFO'),
   $console = hiera('dmlite::accounting::console', true),
-  $ssm_url = hiera('dmlite::accounting::ssm_url', 'https://github.com/apel/ssm/releases/download/2.1.7-1/apel-ssm-2.1.7-1.el6.noarch.rpm'),
+  $ssm_url = hiera('dmlite::accounting::ssm_url', ''),
 
 ) {
 
- if $site_name == '' {
-  fail("'site_name' not defined")
- }
- #install
- package {'python-daemon':
-  ensure => 'installed',
- }   
- package {'python-ldap':
-  ensure => 'installed',
- }
- package {'python-lockfile':
-  ensure => 'installed',
- }
- package {'stomppy':
-  ensure => 'installed',
- }
- package { 'apel-ssm':
-  ensure => 'installed',
-  source => $ssm_url,
-  provider => 'rpm'
- } 
-
- file {'/etc/apel/sender.cfg':
-   ensure  => present,
-   owner   => root,
-   group   => root,
-   content => template('dmlite/ssm/sender.cfg.erb'),
-   require => Package['apel-ssm']
- }
-
- file {"/etc/cron.${cron_interval}/dpm-accounting":
+  if $site_name == '' {
+    fail("'site_name' not defined")
+  }
+
+  # install
+  if $enabled {
+    package {['python-daemon','python-ldap','python-lockfile','stomppy']:
+      ensure => 'installed',
+    }
+  }
+  # apel-ssm also available in UMD repository
+  if $ssm_url == '' {
+    if $facts['os']['family'] == 'RedHat' {
+      $ssm_package_url = $facts['os']['release']['major'] ? {
+        '6' => 'https://github.com/apel/ssm/releases/download/2.3.0-2/apel-ssm-2.3.0-2.el6.noarch.rpm',
+        '7' => 'https://github.com/apel/ssm/releases/download/2.3.0-2/apel-ssm-2.3.0-2.el7.noarch.rpm',
+      }
+    }
+  } else {
+    $ssm_package_url = $ssm_url
+  }
+  if !$ssm_package_url {
+    fail("missing ssm_url on unsupported os ${facts['os']['family']} (${facts['os']['name']} ${facts['os']['release']['major']})")
+  }
+  package { 'apel-ssm':
+    ensure => $enabled ? {
+      true => 'installed',
+      false => absent,
+    },
+    source   => $ssm_package_url,
+    provider => 'rpm'
+  }
+
+  file {'/etc/apel/sender.cfg':
+    ensure  => $enabled ? {
+      true => present,
+      false => absent,
+    },
     owner   => root,
     group   => root,
-    mode    => '0755',
-    content => inline_template("
-#!/bin/sh
+    content => template('dmlite/ssm/sender.cfg.erb'),
+    require => Package['apel-ssm']
+  }
 
-/bin/mkdir -p /var/spool/apel/outgoing/`/bin/date +%Y%m%d` && 
-/usr/share/lcgdm/scripts/star-accounting.py --reportgroups --nsconfig=<%= @nsconfig %> --site=<%= @site_name %> > /var/spool/apel/outgoing/`/bin/date +%Y%m%d`/`date +%Y%m%d%H%M%S` && ssmsend
-"),
-    require => Package['apel-ssm'],
-    ensure => present
+  # do not break in case the new parameters are not defined
+  if $dbuser == '' {
+    $cron_content = inline_template('#!/bin/sh
+/bin/mkdir -p /var/spool/apel/outgoing/`date +%Y%m%d` && /usr/share/dmlite/StAR-accounting/star-accounting.py --reportgroups --nsconfig=<%= @nsconfig %> --site=<%= @site_name %> > /var/spool/apel/outgoing/`date +%Y%m%d`/`date +%Y%m%d%H%M%S` && ssmsend
+')
+  } else {
+    $cron_content = inline_template('#!/bin/sh
+/bin/mkdir -p /var/spool/apel/outgoing/`date +%Y%m%d` && /usr/share/dmlite/StAR-accounting/star-accounting.py --reportgroups --dbhost=<%= @dbhost %> --dbuser=<%= @dbuser %> --dbpwd=<%= @dbpwd %> --nsdbname=<%= @nsdbname %> --dpmdbname=<%= @dpmdbname %> --site=<%= @site_name %> > /var/spool/apel/outgoing/`date +%Y%m%d`/`date +%Y%m%d%H%M%S` && ssmsend
+')
+  }
+
+  file {"/etc/cron.${cron_interval}/dmlite-StAR-accounting":
+    ensure  => $enabled ? {
+      true => present,
+      false => absent,
+    },
+    owner   => root,
+    group   => root,
+    mode    => '0755',
+    content => $cron_content,
+    require => Package['apel-ssm']
+  }
+
+  #purge old cron
+  cron { 'dmlite-star-accounting':
+    ensure => absent,
   }
 
 }
diff --git a/manifests/dav/config.pp b/manifests/dav/config.pp
@@ -34,6 +34,7 @@
   $libdir             = $dmlite::dav::libdir,
   $dav_http_port      = $dmlite::dav::dav_http_port,
   $dav_https_port     = $dmlite::dav::dav_https_port,
+  $enable_srr_cgi     = $dmlite::dav::enable_srr_cgi,
 ) {
   validate_bool($enable_ns)
   validate_bool($enable_disk)

diff --git a/manifests/dav/params.pp b/manifests/dav/params.pp
@@ -33,4 +33,5 @@
     $enable_keep_alive  = hiera('dmlite::dav::params::enable_keep_alive', true)
     $mpm_model          = hiera('dmlite::dav::params::mpm_model', '/usr/sbin/httpd.event')
     $enable_hdfs        = hiera('dmlite::dav::params::enable_hdfs', false)
+    $enable_srr_cgi     = hiera('dmlite::dav::params::enable_srr_cgi', true)
 }
diff --git a/manifests/db/dpm.pp b/manifests/db/dpm.pp
@@ -1,4 +1,5 @@
-class dmlite::db::dpm ($dbuser, $dbpass, $dbhost) inherits dmlite::db::params {
+class dmlite::db::dpm ($dbname, $dbuser, $dbpass, $dbhost) inherits dmlite::db::params {
+  include 'mysql::server'
 
   # the packaged db script explicitly creates the db, we don't want that
   file_line { 'dpm mysql commentcreate':
@@ -7,8 +8,15 @@
     line   => '-- CREATE DATABASE.*',
     path   => '/usr/share/dmlite/dbscripts/dpm_mysql_db.sql'
   }
-
-  mysql::db { $dmlite::db::params::dpm_db:
+  # the packaged db script hardcode  the db name, we don't want that
+  file_line { 'dpm mysql commentuse':
+    ensure => present,
+    match  => 'USE dpm_db.*',
+    line   => '-- USE dpm_db',
+    path   => '/usr/share/dmlite/dbscripts/dpm_mysql_db.sql'
+  }
+
+  mysql::db { $dbname:
     user     => "${dbuser}",
     password => "${dbpass}",
     host     => "${dbhost}",
@@ -24,14 +32,14 @@
             password_hash => mysql_password($dbpass),
             provider      => 'mysql',
         }
-        mysql_grant { "${dbuser}@${::fqdn}/${dmlite::db::params::dpm_db}.*":
+        mysql_grant { "${dbuser}@${::fqdn}/${dbname}.*":
             ensure     => 'present',
             options    => ['GRANT'],
             privileges => ['ALL'],
             provider   => 'mysql',
             user       => "${dbuser}@${::fqdn}",
-            table      => "${dmlite::db::params::dpm_db}.*",
-            require    => [Mysql_database["${dmlite::db::params::dpm_db}"], Mysql_user["${dbuser}@${::fqdn}"], ],
+            table      => "${dbname}.*",
+            require    => [Mysql_database["${dbname}"], Mysql_user["${dbuser}@${::fqdn}"], ],
         }
   } 
 }
diff --git a/manifests/db/ns.pp b/manifests/db/ns.pp
@@ -1,4 +1,5 @@
-class dmlite::db::ns ($flavor , $dbuser, $dbpass, $dbhost) inherits dmlite::db::params {
+class dmlite::db::ns ($flavor , $dbname, $dbuser, $dbpass, $dbhost) inherits dmlite::db::params {
+  include 'mysql::server'
 
   # the packaged db script explicitly creates the db, we don't want that
   file_line { "${flavor} mysql commentcreate":
@@ -7,7 +8,14 @@
     line   => '-- CREATE DATABASE.*',
     path   => "/usr/share/dmlite/dbscripts/cns_mysql_db.sql"
   }
-
+
+  # the packaged db script hardcode  the db name, we don't want that
+  file_line { 'cns mysql commentuse':
+    ensure => present,
+    match  => 'USE cns_db.*',
+    line   => '-- USE cns_db',
+    path   => '/usr/share/dmlite/dbscripts/cns_mysql_db.sql'
+  }
   #workaroundworkaround for missing / db creation
 
   file_line { 'workaround for missing / db creation':
@@ -16,7 +24,7 @@
     path   => "/usr/share/dmlite/dbscripts/cns_mysql_db.sql"
   }
 
-  mysql::db { $dmlite::db::params::ns_db:
+  mysql::db { $dbname:
     user     => "${dbuser}",
     password => "${dbpass}",
     host     => "${dbhost}",
@@ -26,14 +34,14 @@
 
   if $dbhost != 'localhost'  and $dbhost != "${::fqdn}" {
         #create the database grants for the user
-        mysql_grant { "${dbuser}@${::fqdn}/${dmlite::db::params::ns_db}.*":
+        mysql_grant { "${dbuser}@${::fqdn}/${dbname}.*":
             ensure     => 'present',
             options    => ['GRANT'],
             privileges => ['ALL'],
             provider   => 'mysql',
             user       => "${dbuser}@${::fqdn}",
-            table      => "${dmlite::db::params::ns_db}.*",
-            require    => [Mysql_database["${dmlite::db::params::ns_db}"], Mysql_user["${dbuser}@${::fqdn}"], ],
+            table      => "${dbname}.*",
+            require    => [Mysql_database["${dbname}"], Mysql_user["${dbuser}@${::fqdn}"], ],
         }
   }
 

diff --git a/manifests/head.pp b/manifests/head.pp
@@ -80,6 +80,7 @@
     Package['dmlite-dpmhead-domeonly']
     ->
     class{'dmlite::db::dpm':
+      dbname => "${dpm_db}",
       dbuser => "${mysql_username}",
       dbpass => "${mysql_password}",
       dbhost => "${mysql_host}",
@@ -89,6 +90,7 @@
     ->
     class{'dmlite::db::ns': 
       flavor => 'mysql',
+      dbname => "${ns_db}",
       dbuser => "${mysql_username}", 
       dbpass => "${mysql_password}",
       dbhost => "${mysql_host}",
@@ -172,7 +174,6 @@
        }     
        exec{'upgradedb350':
          command => "/bin/sh /usr/share/dmlite/dbscripts/upgrade/DPM_upgrade_mysql \"${mysql_host}\" \"${mysql_username}\" \"${mysql_password}\" \"${dpm_db}\" \"${ns_db}\"",
-         unless => "/bin/sh /usr/share/dmlite/dbscripts/upgrade/check_schema_version \"${mysql_host}\" \"${mysql_username}\" \"${mysql_password}\" \"${dpm_db}\" \"${ns_db}\"",
          require => [ Class['dmlite::db::dpm'], Class['dmlite::db::ns'], Package['dmlite-dpmhead-domeonly']]
        }
      } else {
@@ -184,7 +185,6 @@
        }      
        exec{'upgradedb350':
          command => "/bin/sh /usr/share/dmlite/dbscripts/upgrade/DPM_upgrade_mysql \"${mysql_host}\" \"${mysql_username}\" \"${mysql_password}\" \"${dpm_db}\" \"${ns_db}\" ",
-         unless => "/bin/sh /usr/share/dmlite/dbscripts/upgrade/check_schema_version \"${mysql_host}\" \"${mysql_username}\" \"${mysql_password}\" \"${dpm_db}\" \"${ns_db}\" ",
          require =>  [ Class['lcgdm'],  Package['dmlite-dpmhead']]
        }
      }
@@ -214,6 +214,8 @@
        db_host      => "${mysql_host}",
        db_user      => "${mysql_username}",
        db_password  => "${mysql_password}",
+       cnsdb_name   => "${ns_db}",
+       dpmdb_name   => "${dpm_db}",
        headnode_domeurl => "http://${dpmhost}:1094/domehead",
        restclient_cli_xrdhttpkey => "${token_password}"
      } 

diff --git a/metadata.json b/metadata.json
@@ -31,5 +31,5 @@
   "source": "https://github.com/cern-it-sdc-id/puppet-dmlite",
   "summary": "Package for managing dmlite plugins",
   "tags": [],
-  "version": "1.13.0"
+  "version": "1.13.2"
 }
diff --git a/templates/dav/zlcgdm-dav.conf b/templates/dav/zlcgdm-dav.conf
@@ -5,8 +5,8 @@
 # VirtualHosts. The actual VirtualHost instances are defined at the end of this file.
 #
 
-<% if !@legacy -%>
-# publish WLCG SRR information online (works only with DPM DOME)
+<%if @enable_srr_cgi -%>
+# publish WLCG SRR information online
 ScriptAlias /static/srr "/usr/bin/dpm-storage-summary.cgi"
 <% end -%>