Hello! We are running our annual fundraising. Please consider making a donation if you value this freely available service or want to support people around the world working towards liberatory social change. https://riseup.net/donate.

Commit 58017b63 authored by taggart's avatar taggart
Browse files

start populating with some stuff from mageia and riseup, not functional yet

parent 82126379
Riseup's sympa puppet module
----------------------------
sympa shared puppet module
==========================
puppet module to manage generic aspects of running sympa.
site/install specific config files, templates, scenari are expected to be
managed via a site specific module that references the generic classes
in this module.
Based on existing puppet code from:
Nicolas Vigier <boklm@mars-attacks.org>
WebSVN: http://svnweb.mageia.org/adm/puppet/modules/sympa/
svn co svn://svn.mageia.org/svn/adm/puppet/modules/sympa/
Sarava sympa class, rhatto
https://git.sarava.org/?p=puppet-mail.git;a=blob_plain;f=manifests/sympa.pp;hb=HEAD
Riseup Networks puppet class (not published)
How to use
==========
TODO
TODO
====
* have nagios subclass depend on statd
* create base class
* setup $sympa_use_upstream variable to override using packages
* setup way to specify sympa base dir (/home/sympa, etc)
* figure out how to integrate with database, apache, postfix
* maybe setup an easy way for site specific scenari/templates/topics/etc
to get pulled in?
* boklm defines a bunch of things that allow you to setup lists, etc. Could
be useful in general, need to figure out where to put them
* write an example site class
-- Matt Taggart <taggart@riseup.net> 2011-12
(C) 2011 Riseup Labs
#!/bin/sh
#
# Plugin to monitor number of sympa lists
#
#%# family=contrib
#%# capabilities=autoconf
case $1 in
autoconf|detect)
echo yes
exit 0;;
config)
cat <<'EOF'
graph_title Total open Sympa lists
graph_vlabel total number of lists
graph_category sympa
graph_total Total
lists.label lists
lists.draw AREA
EOF
exit 0;;
esac
cat <<EOF
lists.value `mysql --defaults-file=/etc/mysql/debian.cnf -s --batch --database=sympa --execute "select count(name_list) from list_table where status_list = 'open'"`
EOF
#!/bin/sh
#
# Plugin to monitor sympa spools
#
#%# family=contrib
#%# capabilities=autoconf
# Can be set via environment, but default is /home/sympa/spool
SPOOLDIR=${spooldir:-/usr/local/sympa/spool}
# auth: for storing messages until they have been confirmed
# bounce: for storing incoming bouncing messages
# bounce/bad: rejected messages. Files are created by the sympa.pl daemon
# digest: for storing list's digests before they are sent
# mod: for storing unmoderated messages
# msg: for storing incoming messages (including commands)
# msg/bad: stores bouncing messages for which couldn't determine the original sender. Files are moved there by the bounced.pl daemon.
# distribute: messages ready for distribution. only used if 2 sympa.pl daemons, one for commands, one for messages)
# distribute/bad: rejected messages. Files are created by the sympa.pl process dedicated to message distribution.
# task: for storing all created tasks
# outgoing: sympa.pl dumps in this spool to wait archiving by archived.pl
# outgoing/bad: messages which couldn't be archived. Files are moved there by the archived.pl daemon.
# expire: used by the expire process
# subscribe: presumably for subscription requests, not in docs
case $1 in
autoconf|detect)
if [ -d $SPOOLDIR/ ] ; then
echo yes
exit 0
else
echo "no (spooldir not found)"
exit 1
fi;;
config)
cat <<'EOF'
graph_title Sympa Spool
graph_vlabel Mails in spool
graph_total Total
graph_category sympa
auth.label auth
bounce.label bounce
bounce_bad.label bounce_bad
bulkspool.label bulkspool
bulkspool_remaining.label bulkspool_remaining
bulkmailer.label bulkmailer
bulkmailer_unique.label bulkmailer_unique
digest.label digest
mod.label mod
msg.label msg
msg_bad.label msg_bad
distribute.label distribute
distribute_bad.label distribute_bad
task.label task
outgoing.label outgoing
outgoing_bad.label outgoing_bad
expire.label expire
subscribe.label subscribe
EOF
exit 0;;
esac
cd $SPOOLDIR >/dev/null 2>/dev/null || {
echo "# Cannot cd to $SPOOLDIR"
exit 1
}
mysql="mysql --defaults-file=/etc/mysql/debian.cnf -s --batch --database=sympa --execute"
bulkspool_count=`$mysql "select count(*) from bulkspool_table"`
bulkspool_legit=`$mysql "select count(*) from bulkspool_table left join bulkmailer_table on bulkmailer_table.messagekey_bulkmailer=bulkspool_table.messagekey_bulkspool where bulkmailer_table.messagekey_bulkmailer IS NULL;"`
let "bulkspool_remaining = bulkspool_count - bulkspool_legit"
cat <<EOF
auth.value `(test -d auth && find auth -type f ) | wc -l`
bounce.value `(test -d bounce && find bounce -maxdepth 1 -type f ) | wc -l`
bounce_bad.value `(test -d bounce/bad && find bounce/bad -type f ) | wc -l`
bulkspool.value $bulkspool_count
bulkspool_remaining.value $bulkspool_remaining
bulkmailer.value `$mysql "select count(*) from bulkmailer_table"`
bulkmailer_unique.value `$mysql "select count(distinct messagekey_bulkmailer) from bulkmailer_table"`
digest.value `( test -d digest && find digest -type f ) | wc -l`
mod.value `( test -d mod && find mod -type f ) | wc -l`
msg.value `( test -d msg && find msg -maxdepth 1 -type f ) | wc -l`
msg_bad.value `( test -d msg/bad && find msg/bad -type f ) | wc -l`
distribute.value `( test -d distribute && find distribute -maxdepth 1 -type f ) | wc -l`
distribute_bad.value `(test -d distribute/bad && find distribute/bad -type f ) | wc -l`
task.value `(test -d task && find task -type f ) | wc -l`
outgoing.value `(test -d outgoing && find outgoing -maxdepth 1 -type f ) | wc -l`
outgoing_bad.value `(test -d outgoing/bad && find outgoing/bad -type f ) | wc -l`
expire.value `(test -d expire && find expire -type f ) | wc -l`
subscribe.value `( test -d subscribe && find subscribe -type f ) | wc -l`
EOF
#!/usr/bin/perl -w
use strict;
#
# Plugin to monitor the number of mails processed by sympa
# Heavily based on exim_mailstats
#
# Usage: copy or link into /etc/munin/node.d/
#
# Parameters:
#
# config (required)
# autoconf (optional - used by munin-config)
#
# Config variables:
#
# logfile - Where to find the syslog-information
#
# Magic markers (optional - used by munin-config and some installation
# scripts):
#
#%# family=auto
#%# capabilities=autoconf
my $statefile = "/var/lib/munin/plugin-state/plugin-sympa_stats.state";
my $pos = undef;
my $badFiles = 0;
my $oldFiles = 0;
my $senderNotAllowed = 0;
my $keyToEditors = 0;
my $msgAccepted = 0;
my $msgArchived = 0;
$pos = 0;
my $logfile = $ENV{'logfile'} || '/opt/sympa/log/syslog';
my $rotlogfile = '';
if ( $ARGV[0] and $ARGV[0] eq "autoconf" )
{
if(! -e $logfile) {
print "no (logfile does not exist)\n";
exit(1);
}
if (-r $logfile) {
print "yes\n";
exit 0;
}
else
{
print "no (logfile not readable)\n";
}
exit 1;
}
if (-f "$logfile.0")
{
$rotlogfile = $logfile . ".0";
}
elsif (-f "$logfile.0.gz")
{
$rotlogfile = $logfile . ".0.gz";
}
elsif (-f "$logfile.1")
{
$rotlogfile = $logfile . ".1";
}
elsif (-f "$logfile.1.gz")
{
$rotlogfile = $logfile . ".1.gz";
}
elsif (-f "$logfile.01")
{
$rotlogfile = $logfile . ".01";
}
elsif (-f "$logfile.01.gz")
{
$rotlogfile = $logfile . ".01.gz";
}
if ( $ARGV[0] and $ARGV[0] eq "config" )
{
print "graph_title some sympa-stats\n";
print "graph_args --base 1000 -l 0\n";
print "graph_vlabel messages/\${graph_period}\n";
print "graph_scale no\n";
print "graph_category sympa\n";
print "badFiles.label bad Files\n";
print "badFiles.type DERIVE\n";
print "badFiles.min 0\n";
print "oldFiles.label old Files\n";
print "oldFiles.type DERIVE\n";
print "oldFiles.min 0\n";
print "senderNotAllowed.label rejected\n";
print "senderNotAllowed.type DERIVE\n";
print "senderNotAllowed.min 0\n";
print "keyToEditors.label keyToEditors\n";
print "keyToEditors.type DERIVE\n";
print "keyToEditors.min 0\n";
print "msgAccepted.label msgAccepted\n";
print "msgAccepted.type DERIVE\n";
print "msgAccepted.min 0\n";
print "msgArchived.label msgArchived\n";
print "msgArchived.type DERIVE\n";
print "msgArchived.min 0\n";
exit 0;
}
if (! -f $logfile and ! -f $rotlogfile)
{
print "badFiles.value U\n";
print "oldFiles.value U\n";
print "senderNotAllowed.value U\n";
print "keyToEditors.value U\n";
print "msgAccepted.value U\n";
print "msgArchived.value U\n";
exit 0;
}
if (-f "$statefile")
{
open (IN, "$statefile") or exit 4;
my $in = <IN>;
if ($in =~ /^(\d+):(\d+):(\d+):(\d+):(\d+):(\d+):(\d+)/)
{
($pos, $badFiles, $oldFiles, $senderNotAllowed, $keyToEditors, $msgAccepted, $msgArchived) = ($1, $2, $3, $4, $5, $6, $7);
}
close IN;
}
my $startsize = (stat $logfile)[7];
if (!defined $pos)
{
# Initial run.
$pos = $startsize;
}
if ($startsize < $pos)
{
# Log rotated, read the rest of the old log before starting on the new
parseLogfile ($rotlogfile, $pos, -1);
$pos = 0;
}
parseLogfile ($logfile, $pos, $startsize);
$pos = $startsize;
print "badFiles.value $badFiles\n";
print "oldFiles.value $oldFiles\n";
print "senderNotAllowed.value $senderNotAllowed\n";
print "keyToEditors.value $keyToEditors\n";
print "msgAccepted.value $msgAccepted\n";
print "msgArchived.value $msgArchived\n";
if(-l $statefile) {
die("$statefile is a symbolic link, refusing to touch it.");
}
open (OUT, ">$statefile") or exit 4;
print OUT "$pos:$badFiles:$oldFiles:$senderNotAllowed:$keyToEditors:$msgAccepted:$msgArchived\n";
close OUT;
sub parseLogfile
{
my ($fname, $start, $stop) = @_;
if ($fname =~ m/.gz$/) {
# rotated log is gzipped, unzip a copy
my $tmpfile = $fname;
$tmpfile =~ s:/var/log/sympa/(.*).gz:/var/tmp/$1:;
`rm -f $tmpfile;zcat $fname > $tmpfile`;
$fname = $tmpfile;
}
open (LOGFILE, $fname) or exit 3;
seek (LOGFILE, $start, 0) or exit 2;
while (<LOGFILE>)
{
my $line =$_;
chomp ($line);
if ($line=~/Moving bad file \S+ to bad/)
{
$badFiles++;
}
elsif ($line=~/Deleting old file /)
{
$oldFiles++;
}
elsif ($line=~/rejected\(\) because sender not allowed/)
{
$senderNotAllowed++;
}
elsif ($line=~/Key \S+ for list \S+ from \S+ sent to editors, /)
{
$keyToEditors++;
}
elsif ($line=~/Message for \S+ from \S+ accepted /)
{
$msgAccepted++;
}
elsif ($line=~/Archiving \S+ for list/)
{
$msgArchived++;
}
last if (tell (LOGFILE) == $stop);
}
close(LOGFILE);
}
# vim:syntax=perl
#!/bin/sh
#
# Plugin to monitor number of sympa subscribers
#
#%# family=contrib
#%# capabilities=autoconf
case $1 in
autoconf|detect)
echo yes
exit 0;;
config)
cat <<'EOF'
graph_title Total Sympa subscribers
graph_vlabel total number of subscribers
graph_category sympa
graph_total Total
subscribers.label subscribers
subscribers.draw AREA
EOF
exit 0;;
esac
cat <<EOF
subscribers.value `mysql --defaults-file=/etc/mysql/debian.cnf -s --batch --database=sympa --execute "select count(user_subscriber) from subscriber_table"`
EOF
#!/bin/sh
#
# Plugin to monitor number of sympa users
#
#%# family=contrib
#%# capabilities=autoconf
case $1 in
autoconf|detect)
echo yes
exit 0;;
config)
cat <<'EOF'
graph_title Total unique Sympa users
graph_vlabel total number of user
graph_category sympa
graph_total Total
users.label lists
users.draw AREA
EOF
exit 0;;
esac
cat <<EOF
users.value `mysql --defaults-file=/etc/mysql/debian.cnf -s --batch --database=sympa --execute "select count(email_user) from user_table"`
EOF
#Trying to sort out depends, see
#https://we.riseup.net/riseup+lists/sympa-dependencies
class sympa::debian {
}
class sympa {
case $operatingsystem {
debian: { include sympa::debian }
mageia: { include sympa::mageia }
default: { include sympa::default }
}
if $use_munin {
include sympa::munin
}
if $use_nagios {
include sympa::nagios
}
### still reviewing the below
# sympa script start 5 differents script, I am not
# sure that puppet will correctly handle this
service { "sympa":
ensure => running,
hasstatus => true,
subscribe => [ Package["sympa"], File['/etc/sympa/sympa.conf']]
}
$pgsql_password = extlookup("sympa_pgsql",'x')
$ldap_password = extlookup("sympa_ldap",'x')
postgresql::remote_db_and_user { 'sympa':
password => $pgsql_password,
description => "Sympa database",
}
file { '/etc/sympa/sympa.conf':
ensure => present,
# should be cleaner to have it root owned, but puppet do not support acl
# and in any case, config will be reset if it change
owner => sympa,
group => apache,
mode => 640,
content => template("sympa/sympa.conf"),
require => Package[sympa],
}
file { '/etc/sympa/auth.conf':
ensure => present,
owner => root,
group => root,
mode => 644,
content => template("sympa/auth.conf"),
require => Package[sympa],
notify => Service['httpd'],
}
include apache::mod_fcgid
apache::webapp_other{"sympa":
webapp_file => "sympa/webapp_sympa.conf",
}
apache::vhost_redirect_ssl { "$vhost": }
apache::vhost_base { "$vhost":
use_ssl => true,
content => template("sympa/vhost_ml.conf"),
}
subversion::snapshot { "/etc/sympa/web_tt2":
source => "svn://svn.mageia.org/svn/web/templates/sympa/trunk"
}
file { ["/etc/sympa/lists_xml/",
"/etc/sympa/scenari/",
"/etc/sympa/data_sources/",
"/etc/sympa/search_filters/"]:
ensure => directory,
owner => root,
group => root,
mode => 755,
purge => true,
recurse => true,
force => true,
require => Package[sympa],
}
file { ["/etc/sympa/scenari/subscribe.open_web_only_notify",
"/etc/sympa/scenari/unsubscribe.open_web_only_notify"]:
ensure => present,
owner => root,
group => root,
mode => 755,
source => "puppet:///modules/sympa/scenari/open_web_only_notify",
}
file { ["/etc/sympa/scenari/send.subscriber_moderated"]:
ensure => present,
owner => root,
group => root,
mode => 755,
source => "puppet:///modules/sympa/scenari/subscriber_moderated",
}
file { ["/etc/sympa/scenari/create_list.forbidden"]:
ensure => present,
owner => root,
group => root,
mode => 755,
source => "puppet:///modules/sympa/scenari/forbidden",
}
file { ["/etc/sympa/topics.conf"]:
ensure => present,
owner => root,
group => root,
mode => 755,
source => "puppet:///modules/sympa/topics.conf",
require => Package[sympa],
}
define ldap_search_filter {
file { "/etc/sympa/search_filters/$name.ldap":
ensure => present,
owner => root,
group => root,
mode => 755,
content => template('sympa/search_filters/group.ldap')
}
}
define ldap_group_datasource {
file { "/etc/sympa/data_sources/$name.incl":
ensure => present,
owner => root,
group => root,
mode => 755,
content => template('sympa/data_sources/ldap_group.incl')
}
}
define scenario_sender_ldap_group {
file { "/etc/sympa/scenari/send.restricted_$name":
ensure => present,
owner => root,
group => root,
mode => 755,
content => template('sympa/scenari/sender.ldap_group')
}
}
define scenario_sender_email {
$sender_email_file = regsubst($name,'\@','-at-')
file { "/etc/sympa/scenari/send.restricted_$sender_email_file":
ensure => present,
owner => root,
group => root,
mode => 755,
content => template('sympa/scenari/sender.email')
}
}
# add each group that could be used in a sympa ml either as
# - owner
# - editor ( moderation )
ldap_group_datasource { "mga-sysadmin": }
ldap_group_datasource { "mga-ml_moderators": }
# directory that will hold the list data
# i am not sure of the name ( misc, 09/12/10 )
file { "/var/lib/sympa/expl/":
ensure => directory,
owner => sympa,
group => root,
mode => 755,
require => Package[sympa],
}
}
define list($subject,