Commit c8ce7153 authored by paz's avatar paz

Merge branch '282-external-filters-2' into 'master'

Resolve "Enable adding filters from external directories"

Closes #282

See merge request schleuder/schleuder!167
parents ebda12d0 54bdf457
......@@ -9,4 +9,5 @@ coverage/
/*.gem.sig
/*.tar.gz
/*.tar.gz.sig
Gemfile.lock
spec/list-defaults.yml
......@@ -29,10 +29,15 @@ This project adheres to [Semantic Versioning](http://semver.org/).
* Error messages are converted into human readable text now, instead of giving their class-name. (#338)
* Require mail-gpg >= 0.3.3, which fixes a bug that let some equal-signs disappear under specific circumstances. (#287)
### Known issues
* With the current used mail library version schleuder uses, there are certain malformed emails that can't be parsed. See #334 for background. This will be fixed in future releases of the mail library.
### Added
* Enable to load external filters, similar to how we allow external plugins. (#282)
### Changed
* Use schleuder.org as website and team@schleuder.org as contact email.
......
......@@ -7,6 +7,17 @@ listlogs_dir: /var/lib/schleuder/lists
# Schleuder reads plugins also from this directory.
plugins_dir: /etc/schleuder/plugins
# Schleuder reads filters also from this directory path,
# in the specific pre_decryption or post_decryption subdirectory.
# Filter files must follow the following convention for the
# filename: \d+_a_name.rb
# Where \d+ is any number, that defines the place in the
# list of filters and a_name must match the method name
# of the filter.
# The built-in filters are using round numbers for their
# positioning within the list. Increased by ten.
filters_dir: /usr/local/lib/schleuder/filters
# How verbose should Schleuder log to syslog? (list-specific messages are written to the list's log-file).
log_level: warn
......
......@@ -46,9 +46,6 @@ Dir["#{libdir}/schleuder/plugins/*.rb"].each do |file|
require file
end
require 'schleuder/filters_runner'
Dir["#{libdir}/schleuder/filters/*.rb"].each do |file|
require file
end
Dir["#{libdir}/schleuder/validators/*.rb"].each do |file|
require file
end
......
......@@ -11,6 +11,7 @@ module Schleuder
'lists_dir' => '/var/lib/schleuder/lists',
'listlogs_dir' => '/var/lib/schleuder/lists',
'plugins_dir' => '/etc/schleuder/plugins',
'filters_dir' => '/usr/local/lib/schleuder/filters',
'log_level' => 'warn',
'superadmin' => 'root@localhost',
'keyserver' => 'hkp://pool.sks-keyservers.net',
......@@ -58,6 +59,10 @@ module Schleuder
instance.config['plugins_dir']
end
def self.filters_dir
instance.config['filters_dir']
end
def self.database
instance.config['database'][ENV['SCHLEUDER_ENV']]
end
......
module Schleuder
module Filters
def self.receive_encrypted_only(list, mail)
if list.receive_encrypted_only? && ! mail.was_encrypted?
list.logger.info "Rejecting mail as unencrypted"
return Errors::MessageUnencrypted.new
end
end
def self.receive_signed_only(list, mail)
if list.receive_signed_only? && ! mail.was_validly_signed?
list.logger.info "Rejecting mail as unsigned"
return Errors::MessageUnsigned.new
end
end
def self.receive_authenticated_only(list, mail)
if list.receive_authenticated_only? && ( ! mail.was_encrypted? || ! mail.was_validly_signed? )
list.logger.info "Rejecting mail as unauthenticated"
return Errors::MessageUnauthenticated.new
end
end
def self.receive_from_subscribed_emailaddresses_only(list, mail)
if list.receive_from_subscribed_emailaddresses_only? && list.subscriptions.where(email: mail.from.first).blank?
list.logger.info "Rejecting mail as not from subscribed address."
return Errors::MessageSenderNotSubscribed.new
end
end
def self.receive_admin_only(list, mail)
if list.receive_admin_only? && ( ! mail.was_validly_signed? || ! mail.signer.admin? )
list.logger.info "Rejecting mail as not from admin."
return Errors::MessageNotFromAdmin.new
end
end
end
end
module Schleuder
module Filters
def self.receive_admin_only(list, mail)
if list.receive_admin_only? && ( ! mail.was_validly_signed? || ! mail.signer.admin? )
list.logger.info "Rejecting mail as not from admin."
return Errors::MessageNotFromAdmin.new
end
end
end
end
module Schleuder
module Filters
def self.receive_authenticated_only(list, mail)
if list.receive_authenticated_only? && ( ! mail.was_encrypted? || ! mail.was_validly_signed? )
list.logger.info "Rejecting mail as unauthenticated"
return Errors::MessageUnauthenticated.new
end
end
end
end
module Schleuder
module Filters
def self.receive_signed_only(list, mail)
if list.receive_signed_only? && ! mail.was_validly_signed?
list.logger.info "Rejecting mail as unsigned"
return Errors::MessageUnsigned.new
end
end
end
end
module Schleuder
module Filters
def self.receive_encrypted_only(list, mail)
if list.receive_encrypted_only? && ! mail.was_encrypted?
list.logger.info "Rejecting mail as unencrypted"
return Errors::MessageUnencrypted.new
end
end
end
end
module Schleuder
module Filters
def self.receive_from_subscribed_emailaddresses_only(list, mail)
if list.receive_from_subscribed_emailaddresses_only? && list.subscriptions.where(email: mail.from.first).blank?
list.logger.info "Rejecting mail as not from subscribed address."
return Errors::MessageSenderNotSubscribed.new
end
end
end
end
......@@ -9,8 +9,8 @@ module Schleuder
# This problem seems to be in fact related to the use of Microsoft
# Exchange. Accordingly, check if the headers contain 'X-MS-Exchange'.
# See #211, #246, #331 and #333 for background.
def self.fix_exchange_messages!(list, mail)
if mail.header_fields.any?{|f| f.name =~ /^X-MS-Exchange-/i } &&
def self.fix_exchange_messages(list, mail)
if mail.header_fields.any?{|f| f.name =~ /^X-MS-Exchange-/i } &&
!mail[:content_type].blank? &&
mail[:content_type].content_type == 'multipart/mixed' && mail.parts.size > 2 &&
mail.parts[0][:content_type].content_type == 'text/plain' &&
......
module Schleuder
module Filters
def self.strip_html_from_alternative!(list, mail)
def self.strip_html_from_alternative(list, mail)
if mail[:content_type].blank? ||
mail[:content_type].content_type != 'multipart/alternative' ||
! mail.to_s.include?('BEGIN PGP ')
......
module Schleuder
module Filters
class Runner
# To define priority sort this.
# The method `setup` parses, decrypts etc.
# the mail sent to the list. So before
# calling setup we do all the things
# that won't require e.g. validation of
# the sender.
PRE_SETUP_FILTERS = %w[
forward_bounce_to_admins
forward_all_incoming_to_admins
send_key
fix_exchange_messages!
strip_html_from_alternative!
]
# message size must be checked after
# decryption as gpg heavily compresses
# messages.
POST_SETUP_FILTERS = %w[
request
max_message_size
forward_to_owner
receive_admin_only
receive_authenticated_only
receive_signed_only
receive_encrypted_only
receive_from_subscribed_emailaddresses_only
]
attr_reader :list, :filter_type
attr_reader :list
def initialize(list)
def initialize(list, filter_type)
@list = list
@filter_type = filter_type
end
def run(mail, filters)
def run(mail)
filters.map do |cmd|
list.logger.debug "Calling filter #{cmd}"
response = Filters.send(cmd, list, mail)
......@@ -48,8 +22,12 @@ module Schleuder
end
nil
end
private
def filters
@filters ||= load_filters
end
private
def stop?(response)
response.kind_of?(StandardError)
end
......@@ -78,6 +56,38 @@ module Schleuder
list.logger.notify_admin reason, original_message, I18n.t('notice')
end
end
def load_filters
list.logger.debug "Loading #{filter_type}_decryption filters"
sorted_filters.map do |filter_name|
require all_filter_files[filter_name]
filter_name.split('_',2).last
end
end
def sorted_filters
@sorted_filters ||= all_filter_files.keys.sort do |a,b|
a.split('_',2).first.to_i <=> b.split('_',2).first.to_i
end
end
def all_filter_files
@all_filter_files ||= begin
files_in_filter_dirs = Dir[*filter_dirs]
files_in_filter_dirs.inject({}) do |res,file|
filter_name = File.basename(file,'.rb')
res[filter_name] = file
res
end
end
end
def filter_dirs
@filter_dirs ||= [File.join(File.dirname(__FILE__),"filters"),
Schleuder::Conf.filters_dir].map do |d|
File.join(d,"#{filter_type}_decryption/[0-9]*_*.rb")
end
end
end
end
end
......@@ -7,7 +7,7 @@ module Schleuder
logger.info "Parsing incoming email."
@mail = Mail.create_message_to_list(msg, recipient, list)
error = run_filters(Filters::Runner::PRE_SETUP_FILTERS)
error = run_filters('pre')
return error if error
begin
......@@ -18,7 +18,7 @@ module Schleuder
return Errors::DecryptionFailed.new(list)
end
error = run_filters(Filters::Runner::POST_SETUP_FILTERS)
error = run_filters('post')
return error if error
if ! @mail.was_validly_signed?
......@@ -56,8 +56,8 @@ module Schleuder
@list
end
def run_filters(filters)
error = filters_runner.run(@mail, filters)
def run_filters(filter_type)
error = filters_runner(filter_type).run(@mail)
if error
if list.bounces_notify_admins?
text = "#{I18n.t('.bounces_notify_admins')}\n\n#{error}"
......@@ -68,8 +68,19 @@ module Schleuder
end
end
def filters_runner
@filters_runner ||= Filters::Runner.new(list)
def filters_runner(filter_type)
if filter_type == 'pre'
filters_runner_pre_decryption
else
filters_runner_post_decryption
end
end
def filters_runner_pre_decryption
@filters_runner_pre_decryption ||= Filters::Runner.new(list,'pre')
end
def filters_runner_post_decryption
@filters_runner_post_decryption ||= Filters::Runner.new(list,'post')
end
def logger
......
module Schleuder::Filters
def self.post_example(list, mail)
end
end
module Schleuder::Filters
def self.example(list, mail)
end
end
module Schleuder::Filters
def self.never_show_up(list, mail)
end
end
module Schleuder::Filters
def self.post_example(list,mail)
end
end
module Schleuder::Filters
def self.late_example(list, mail)
end
end
module Schleuder::Filters
def self.example(list, mail)
end
end
module Schleuder::Filters
def self.early_example(list, mail)
end
end
......@@ -10,21 +10,24 @@ module Schleuder::Filters
end
describe Schleuder::Filters::Runner do
let(:subject) do
let(:list) do
# setup the list with an admin that can be notified
list = create(:list, send_encrypted_only: false)
list.subscribe("schleuder@example.org", nil, true)
Schleuder::Filters::Runner.new(list)
list
end
let(:pre_filters) { Schleuder::Filters::Runner.new(list,'pre') }
let(:post_filters){ Schleuder::Filters::Runner.new(list,'post') }
it { is_expected.to respond_to :run }
it { expect(pre_filters).to respond_to :run }
context '#run' do
it 'runs the filters' do
mail = Mail.new
expect(Schleuder::Filters).to receive(:dummy).once
expect(Schleuder::Filters).to_not receive(:stop)
expect(subject.run(mail,['dummy'])).to be_nil
expect(pre_filters).to receive(:filters).and_return(['dummy'])
expect(pre_filters.run(mail)).to be_nil
end
it 'stops on a StandardError and returns error' do
......@@ -32,16 +35,18 @@ describe Schleuder::Filters::Runner do
error = StandardError.new
expect(Schleuder::Filters).to_not receive(:dummy)
expect(Schleuder::Filters).to receive(:stop).once { error }
expect(subject.run(mail,['stop','dummy'])).to eql(error)
expect(pre_filters).to receive(:filters).and_return(['stop','dummy'])
expect(pre_filters.run(mail)).to eql(error)
expect(Mail::TestMailer.deliveries.first).to be_nil
end
it 'stops on a StandardError and will notify admins' do
mail = Mail.new
error = StandardError.new
subject.list.bounces_drop_all = true
pre_filters.list.bounces_drop_all = true
expect(Schleuder::Filters).to_not receive(:dummy)
expect(Schleuder::Filters).to receive(:stop).once { error }
expect(subject.run(mail,['stop','dummy'])).to be_nil
expect(pre_filters).to receive(:filters).and_return(['stop','dummy'])
expect(pre_filters.run(mail)).to be_nil
expect(Mail::TestMailer.deliveries.first).to_not be_nil
end
it 'stops on a StandardError and will notify on headers match' do
......@@ -50,8 +55,97 @@ describe Schleuder::Filters::Runner do
mail['X-SPAM-FLAG'] = 'TRUE'
expect(Schleuder::Filters).to_not receive(:dummy)
expect(Schleuder::Filters).to receive(:stop).once { error }
expect(subject.run(mail,['stop','dummy'])).to be_nil
expect(pre_filters).to receive(:filters).and_return(['stop','dummy'])
expect(pre_filters.run(mail)).to be_nil
expect(Mail::TestMailer.deliveries.first).to_not be_nil
end
end
context 'loading filters' do
it 'loads filters from built-in filters_dir sorts them' do
Schleuder::Conf.instance.config['filters_dir'] = File.join(Dir.pwd,'spec/fixtures/no_filters')
expect(pre_filters.filters).to eq [
'forward_bounce_to_admins',
'forward_all_incoming_to_admins',
'send_key',
'fix_exchange_messages',
'strip_html_from_alternative'
]
expect(post_filters.filters).to eq [
'request',
'max_message_size',
'forward_to_owner',
'receive_admin_only',
'receive_authenticated_only',
'receive_signed_only',
'receive_encrypted_only',
'receive_from_subscribed_emailaddresses_only',
]
end
it 'loads custom filters from filters_dir and sorts them in, ignores filter not following convention' do
Schleuder::Conf.instance.config['filters_dir'] = File.join(Dir.pwd,'spec/fixtures/filters')
expect(pre_filters.filters).to eq [
'forward_bounce_to_admins',
'forward_all_incoming_to_admins',
'example',
'send_key',
'fix_exchange_messages',
'strip_html_from_alternative'
]
expect(post_filters.filters).to eq [
'request',
'max_message_size',
'forward_to_owner',
'receive_admin_only',
'receive_authenticated_only',
'receive_signed_only',
'receive_encrypted_only',
'post_example',
'receive_from_subscribed_emailaddresses_only',
]
end
it 'loads custom filters from filters_dir and sorts them in with missing dir' do
Schleuder::Conf.instance.config['filters_dir'] = File.join(Dir.pwd,'spec/fixtures/filters_without_pre')
expect(pre_filters.filters).to eq [
'forward_bounce_to_admins',
'forward_all_incoming_to_admins',
'send_key',
'fix_exchange_messages',
'strip_html_from_alternative'
]
expect(post_filters.filters).to eq [
'post_example',
'request',
'max_message_size',
'forward_to_owner',
'receive_admin_only',
'receive_authenticated_only',
'receive_signed_only',
'receive_encrypted_only',
'receive_from_subscribed_emailaddresses_only',
]
end
it 'loads custom filters from filters_dir even with non-2-digit priority' do
Schleuder::Conf.instance.config['filters_dir'] = File.join(Dir.pwd,'spec/fixtures/more_filters')
expect(pre_filters.filters).to eq [
'early_example',
'forward_bounce_to_admins',
'forward_all_incoming_to_admins',
'example',
'send_key',
'fix_exchange_messages',
'strip_html_from_alternative',
'late_example'
]
expect(post_filters.filters).to eq [
'request',
'max_message_size',
'forward_to_owner',
'receive_admin_only',
'receive_authenticated_only',
'receive_signed_only',
'receive_encrypted_only',
'receive_from_subscribed_emailaddresses_only',
]
end
end
end
require "spec_helper"
# make sure we have the filters loaded, as they will be loaded lazily within the code
Dir[File.join(File.dirname(__FILE__),'../../../lib/schleuder/filters/*/*.rb')].each do |file|
require file
end
describe Schleuder::Filters do
context '.fix_exchange_messages!' do
context '.fix_exchange_messages' do
it "fixes pgp/mime-messages that were mangled by Exchange" do
message = Mail.read("spec/fixtures/mails/exchange.eml")
Schleuder::Filters.fix_exchange_messages!(nil, message)
Schleuder::Filters.fix_exchange_messages(nil, message)
expect(message[:content_type].content_type).to eql("multipart/encrypted")
end
it "works with a text/plain message" do
message = Mail.read("spec/fixtures/mails/exchange_no_parts.eml")
Schleuder::Filters.fix_exchange_messages!(nil, message)
Schleuder::Filters.fix_exchange_messages(nil, message)
expect(message[:content_type].content_type).to eql("text/plain")
end
end
context '.strip_html_from_alternative!' do
context '.strip_html_from_alternative' do
it "strips HTML-part from multipart/alternative-message that contains ascii-armored PGP-data" do
list = create(:list)
mail = Mail.new
......@@ -28,7 +32,7 @@ describe Schleuder::Filters do
mail.html_part = "<p>#{content}</p>"
mail.subject = "test"
Schleuder::Filters.strip_html_from_alternative!(list, mail)
Schleuder::Filters.strip_html_from_alternative(list, mail)
expect(mail[:content_type].content_type).to eql("multipart/mixed")
expect(mail.parts.size).to be(1)
......@@ -45,7 +49,7 @@ describe Schleuder::Filters do
mail.html_part = "<p>#{content}</p>"
mail.subject = "test"
Schleuder::Filters.strip_html_from_alternative!(nil, mail)
Schleuder::Filters.strip_html_from_alternative(nil, mail)
expect(mail[:content_type].content_type).to eql("multipart/alternative")
expect(mail.parts.size).to be(2)
......@@ -61,7 +65,7 @@ describe Schleuder::Filters do
mail.body = "blabla"
mail.subject = "test"
Schleuder::Filters.strip_html_from_alternative!(nil, mail)
Schleuder::Filters.strip_html_from_alternative(nil, mail)
expect(mail[:content_type]).to be_nil
expect(mail.parts.size).to be(0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment