Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • anon_resource
  • bug/add_schema_to_manifest
  • bug/credential_factories
  • bug/cryptobackend
  • bug/fail-gracefully-no-syncer
  • bug/raise-error
  • bug/remove-unicode-conversion
  • debian/experimental
  • debian/platform-0.5
  • debian/platform-0.6
  • debian/platform-0.7
  • debian/platform-0.8
  • debian/platform-0.9
  • debug_threads
  • deploy
  • develop
  • docs/incoming_box_spec
  • docs/migrationpolicy
  • feat/openssl_scrypt
  • feature/async-api
  • feature/blobs_stub
  • feature/crazy_blobs
  • feature/enc-sync-transitional-db
  • feature/streaming_encrypter
  • feature/toggleblobs
  • feature/unauth_banner
  • feature_priorities
  • fix-pep8
  • fix_auth
  • master
  • pkg/add-routes
  • pkg/unpin_everything
  • refactor/rename
  • release/0.6.x
  • release/0.7.x
  • release/0.8.x
  • release/bitmask-0.9.x-alpha
  • test_deploy
  • tests/include-pep8-and-cov
  • varacpkg
  • 0.2.1
  • 0.2.2
  • 0.2.3
  • 0.3.0
  • 0.3.1
  • 0.3.2
  • 0.4.0
  • 0.4.1
  • 0.4.2
  • 0.4.3
  • 0.4.4
  • 0.4.5
  • 0.5.0
  • 0.5.1
  • 0.5.2
  • 0.6.0
  • 0.6.1
  • 0.6.2
  • 0.6.3
  • 0.6.4
  • 0.6.5
  • 0.7.0
  • 0.7.1
  • 0.7.2
  • 0.7.3
  • 0.7.4
  • 0.8.0
  • 0.8.1
68 results

Target

Select target project
  • drebs/soledad
  • shyba/soledad
  • kali/soledad
  • micah/soledad
  • efkin/soledad
  • vdegou/soledad
  • cyberdrudge/soledad
  • jrabbit/soledad
8 results
Select Git revision
  • anon_resource
  • bug/add_schema_to_manifest
  • bug/credential_factories
  • bug/cryptobackend
  • bug/fail-gracefully-no-syncer
  • bug/raise-error
  • bug/remove-unicode-conversion
  • debian/experimental
  • debian/platform-0.5
  • debian/platform-0.6
  • debian/platform-0.7
  • debian/platform-0.8
  • debian/platform-0.9
  • debug_threads
  • deploy
  • develop
  • docs/incoming_box_spec
  • docs/migrationpolicy
  • feat/openssl_scrypt
  • feature/async-api
  • feature/blobs_stub
  • feature/crazy_blobs
  • feature/enc-sync-transitional-db
  • feature/streaming_encrypter
  • feature/toggleblobs
  • feature/unauth_banner
  • feature_priorities
  • fix-pep8
  • fix_auth
  • master
  • pkg/add-routes
  • pkg/unpin_everything
  • refactor/rename
  • release/0.6.x
  • release/0.7.x
  • release/0.8.x
  • release/bitmask-0.9.x-alpha
  • test_deploy
  • tests/include-pep8-and-cov
  • varacpkg
  • 0.2.1
  • 0.2.2
  • 0.2.3
  • 0.3.0
  • 0.3.1
  • 0.3.2
  • 0.4.0
  • 0.4.1
  • 0.4.2
  • 0.4.3
  • 0.4.4
  • 0.4.5
  • 0.5.0
  • 0.5.1
  • 0.5.2
  • 0.6.0
  • 0.6.1
  • 0.6.2
  • 0.6.3
  • 0.6.4
  • 0.6.5
  • 0.7.0
  • 0.7.1
  • 0.7.2
  • 0.7.3
  • 0.7.4
  • 0.8.0
  • 0.8.1
68 results
Show changes

Commits on Source 31

Showing
with 1021 additions and 92 deletions
# -*- coding: utf-8 -*-
# _blobs.py
# Copyright (C) 2017 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Clientside BlobBackend Storage.
"""
from copy import copy
from urlparse import urljoin
import os.path
import uuid
from io import BytesIO
from functools import partial
from twisted.logger import Logger
from twisted.enterprise import adbapi
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
import treq
from leap.soledad.client.sqlcipher import SQLCipherOptions
from leap.soledad.client import pragmas
from _crypto import DocInfo, BlobEncryptor, BlobDecryptor
logger = Logger()
class ConnectionPool(adbapi.ConnectionPool):
def insertAndGetLastRowid(self, *args, **kwargs):
"""
Execute an SQL query and return the last rowid.
See: https://sqlite.org/c3ref/last_insert_rowid.html
"""
return self.runInteraction(
self._insertAndGetLastRowid, *args, **kwargs)
def _insertAndGetLastRowid(self, trans, *args, **kw):
trans.execute(*args, **kw)
return trans.lastrowid
def blob(self, table, column, irow, flags):
"""
Open a BLOB for incremental I/O.
Return a handle to the BLOB that would be selected by:
SELECT column FROM table WHERE rowid = irow;
See: https://sqlite.org/c3ref/blob_open.html
:param table: The table in which to lookup the blob.
:type table: str
:param column: The column where the BLOB is located.
:type column: str
:param rowid: The rowid of the BLOB.
:type rowid: int
:param flags: If zero, BLOB is opened for read-only. If non-zero,
BLOB is opened for RW.
:type flags: int
:return: A BLOB handle.
:rtype: pysqlcipher.dbapi.Blob
"""
return self.runInteraction(self._blob, table, column, irow, flags)
def _blob(self, trans, table, column, irow, flags):
# TODO: should not use transaction private variable here
handle = trans._connection.blob(table, column, irow, flags)
return handle
class DecrypterBuffer(object):
def __init__(self, doc_id, rev, secret):
self.decrypter = None
self.buffer = BytesIO()
self.doc_info = DocInfo(doc_id, rev)
self.secret = secret
self.d = None
def write(self, data):
if not self.decrypter:
self.buffer.write(data)
self.decrypter = BlobDecryptor(
self.doc_info, self.buffer,
secret=self.secret,
armor=True,
start_stream=False)
self.d = self.decrypter.decrypt()
else:
self.decrypter.write(data)
def close(self):
if self.d:
self.d.addCallback(lambda result: (result, self.decrypter.size))
return self.d
class BlobManager(object):
"""
Ideally, the decrypting flow goes like this:
- GET a blob from remote server.
- Decrypt the preamble
- Allocate a zeroblob in the sqlcipher sink
- Mark the blob as unusable (ie, not verified)
- Decrypt the payload incrementally, and write chunks to sqlcipher
** Is it possible to use a small buffer for the aes writer w/o
** allocating all the memory in openssl?
- Finalize the AES decryption
- If preamble + payload verifies correctly, mark the blob as usable
"""
def __init__(self, local_path, remote, key, secret, user):
self.local = SQLiteBlobBackend(local_path, key)
self.remote = remote
self.secret = secret
self.user = user
@defer.inlineCallbacks
def put(self, doc):
fd = doc.blob_fd
# TODO this is a tee really, but ok... could do db and upload
# concurrently. not sure if we'd gain something.
yield self.local.put(doc.blob_id, fd)
fd.seek(0)
yield self._encrypt_and_upload(doc.blob_id, doc.doc_id, doc.rev, fd)
@defer.inlineCallbacks
def get(self, blob_id, doc_id, rev):
local_blob = yield self.local.get(blob_id)
if local_blob:
logger.info("Found blob in local database: %s" % blob_id)
defer.returnValue(local_blob)
result = yield self._download_and_decrypt(blob_id, doc_id, rev)
if not result:
defer.returnValue(None)
blob, size = result
if blob:
logger.info("Got decrypted blob of type: %s" % type(blob))
blob.seek(0)
yield self.local.put(blob_id, blob, size=size)
blob.seek(0)
defer.returnValue(blob)
else:
# XXX we shouldn't get here, but we will...
# lots of ugly error handling possible:
# 1. retry, might be network error
# 2. try later, maybe didn't finished streaming
# 3.. resignation, might be error while verifying
logger.error('sorry, dunno what happened')
@defer.inlineCallbacks
def _encrypt_and_upload(self, blob_id, doc_id, rev, fd):
# TODO ------------------------------------------
# this is wrong, is doing 2 stages.
# the crypto producer can be passed to
# the uploader and react as data is written.
# try to rewrite as a tube: pass the fd to aes and let aes writer
# produce data to the treq request fd.
# ------------------------------------------------
logger.info("Staring upload of blob: %s" % blob_id)
doc_info = DocInfo(doc_id, rev)
uri = urljoin(self.remote, self.user + "/" + blob_id)
crypter = BlobEncryptor(doc_info, fd, secret=self.secret,
armor=True)
fd = yield crypter.encrypt()
yield treq.put(uri, data=fd)
logger.info("Finished upload: %s" % (blob_id,))
@defer.inlineCallbacks
def _download_and_decrypt(self, blob_id, doc_id, rev):
logger.info("Staring download of blob: %s" % blob_id)
# TODO this needs to be connected in a tube
uri = self.remote + self.user + '/' + blob_id
buf = DecrypterBuffer(doc_id, rev, self.secret)
data = yield treq.get(uri)
if data.code == 404:
logger.warn("Blob not found in server: %s" % blob_id)
defer.returnValue(None)
# incrementally collect the body of the response
yield treq.collect(data, buf.write)
fd, size = yield buf.close()
logger.info("Finished download: (%s, %d)" % (blob_id, size))
defer.returnValue((fd, size))
class SQLiteBlobBackend(object):
def __init__(self, path, key=None):
self.path = os.path.abspath(
os.path.join(path, 'soledad_blob.db'))
if not key:
raise ValueError('key cannot be None')
backend = 'pysqlcipher.dbapi2'
opts = SQLCipherOptions('/tmp/ignored', key)
pragmafun = partial(pragmas.set_init_pragmas, opts=opts)
openfun = _sqlcipherInitFactory(pragmafun)
self.dbpool = ConnectionPool(
backend, self.path, check_same_thread=False, timeout=5,
cp_openfun=openfun, cp_min=1, cp_max=2, cp_name='blob_pool')
@defer.inlineCallbacks
def put(self, blob_id, blob_fd, size=None):
logger.info("Saving blob in local database...")
insert = 'INSERT INTO blobs (blob_id, payload) VALUES (?, zeroblob(?))'
irow = yield self.dbpool.insertAndGetLastRowid(insert, (blob_id, size))
handle = yield self.dbpool.blob('blobs', 'payload', irow, 1)
blob_fd.seek(0)
# XXX I have to copy the buffer here so that I'm able to
# return a non-closed file to the caller (blobmanager.get)
# FIXME should remove this duplication!
# have a look at how treq does cope with closing the handle
# for uploading a file
producer = FileBodyProducer(copy(blob_fd))
done = yield producer.startProducing(handle)
logger.info("Finished saving blob in local database.")
defer.returnValue(done)
@defer.inlineCallbacks
def get(self, blob_id):
# TODO we can also stream the blob value using sqlite
# incremental interface for blobs - and just return the raw fd instead
select = 'SELECT payload FROM blobs WHERE blob_id = ?'
result = yield self.dbpool.runQuery(select, (blob_id,))
if result:
defer.returnValue(BytesIO(str(result[0][0])))
def _init_blob_table(conn):
maybe_create = (
"CREATE TABLE IF NOT EXISTS "
"blobs ("
"blob_id PRIMARY KEY, "
"payload BLOB)")
conn.execute(maybe_create)
def _sqlcipherInitFactory(fun):
def _initialize(conn):
fun(conn)
_init_blob_table(conn)
return _initialize
class BlobDoc(object):
# TODO probably not needed, but convenient for testing for now.
def __init__(self, doc_id, rev, content, blob_id=None):
self.doc_id = doc_id
self.rev = rev
self.is_blob = True
self.blob_fd = content
if blob_id is None:
blob_id = uuid.uuid4().get_hex()
self.blob_id = blob_id
#
# testing facilities
#
@defer.inlineCallbacks
def testit(reactor):
# configure logging to stdout
from twisted.python import log
import sys
log.startLogging(sys.stdout)
# parse command line arguments
import argparse
usage = "\n mkdir /tmp/blobs/user && cd server/src/leap/soledad/server/ && python _blobs.py" \
"\n python _blobs.py upload /path/to/file blob_id" \
"\n python _blobs.py download blob_id"
parser = argparse.ArgumentParser(usage=usage)
subparsers = parser.add_subparsers(help='sub-command help', dest='action')
# parse upload command
parser_upload = subparsers.add_parser(
'upload', help='upload blob and bypass local db')
parser_upload.add_argument('payload')
parser_upload.add_argument('blob_id')
# parse download command
parser_download = subparsers.add_parser(
'download', help='download blob and bypass local db')
parser_download.add_argument('blob_id')
# parse put command
parser_put = subparsers.add_parser(
'put', help='put blob in local db and upload')
parser_put.add_argument('payload')
parser_put.add_argument('blob_id')
# parse get command
parser_get = subparsers.add_parser(
'get', help='get blob from local db, get if needed')
parser_get.add_argument('blob_id')
# parse arguments
args = parser.parse_args()
# TODO convert these into proper unittests
def _manager():
manager = BlobManager(
'/tmp/blobs', 'http://localhost:9000/',
'A' * 32, 'secret', 'user')
return manager
@defer.inlineCallbacks
def _upload(blob_id, payload):
logger.info(":: Starting upload only: %s" % str((blob_id, payload)))
manager = _manager()
with open(payload, 'r') as fd:
yield manager._encrypt_and_upload(blob_id, 'mydoc', '1', fd)
logger.info(":: Finished upload only: %s" % str((blob_id, payload)))
@defer.inlineCallbacks
def _download(blob_id):
logger.info(":: Starting download only: %s" % blob_id)
manager = _manager()
result = yield manager._download_and_decrypt(blob_id, 'mydoc', '1')
logger.info(":: Result of download: %s" % str(result))
if result:
fd, _ = result
logger.info(":: Content of blob %s: %s" % (blob_id, fd.getvalue()))
logger.info(":: Finished download only: %s" % blob_id)
@defer.inlineCallbacks
def _put(blob_id, payload):
logger.info(":: Starting full put: %s" % blob_id)
manager = _manager()
with open(payload) as fd:
doc = BlobDoc('mydoc', '1', fd, blob_id=blob_id)
result = yield manager.put(doc)
logger.info(":: Result of put: %s" % str(result))
logger.info(":: Finished full put: %s" % blob_id)
@defer.inlineCallbacks
def _get(blob_id):
logger.info(":: Starting full get: %s" % blob_id)
manager = _manager()
fd = yield manager.get(blob_id, 'mydoc', '1')
if fd:
logger.info(":: Result of get: " + fd.getvalue())
logger.info(":: Finished full get: %s" % blob_id)
if args.action == 'upload':
yield _upload(args.blob_id, args.payload)
elif args.action == 'download':
yield _download(args.blob_id)
elif args.action == 'put':
yield _put(args.blob_id, args.payload)
elif args.action == 'get':
yield _get(args.blob_id)
if __name__ == '__main__':
from twisted.internet.task import react
react(testit)
......@@ -16,12 +16,45 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Cryptographic operations for the soledad client
Cryptographic operations for the soledad client.
This module implements streaming crypto operations.
It replaces the old client.crypto module, that will be deprecated in soledad
0.12.
The algorithm for encryptig and decrypting is as follow:
The KEY is a 32 bytes value.
The PREAMBLE is a packed_structure with encryption metadata.
The SEPARATOR is a space.
Encryption
----------
ciphertext = b64_encode(packed_preamble)
+ SEPARATOR
+ b64(AES_GCM(ciphertext) + tag)
Decryption
----------
PREAMBLE + SEPARATOR + PAYLOAD
Ciphertext and Tag CAN be encoded in b64 (armor=True) or raw (False)
check_preamble(b64_decode(ciphertext.split(SEPARATOR)[0])
PAYLOAD = ciphertext + tag
decrypt(PAYLOAD)
"""
import binascii
import base64
import hashlib
import warnings
import hmac
import os
import re
......@@ -42,14 +75,16 @@ from cryptography.hazmat.backends.multibackend import MultiBackend
from cryptography.hazmat.backends.openssl.backend \
import Backend as OpenSSLBackend
from zope.interface import implements
from zope.interface import implementer
SECRET_LENGTH = 64
SEPARATOR = ' '
CRYPTO_BACKEND = MultiBackend([OpenSSLBackend()])
PACMAN = struct.Struct('2sbbQ16s255p255p')
PACMAN = struct.Struct('2sbbQ16s255p255pQ')
LEGACY_PACMAN = struct.Struct('2sbbQ16s255p255p')
BLOB_SIGNATURE_MAGIC = '\x13\x37'
......@@ -171,28 +206,49 @@ def decrypt_sym(data, key, iv, method=ENC_METHOD.aes_256_gcm):
return plaintext
# TODO maybe rename this to Encryptor, since it will be used by blobs an non
# blobs in soledad.
class BlobEncryptor(object):
"""
Produces encrypted data from the cleartext data associated with a given
SoledadDocument using AES-256 cipher in GCM mode.
The production happens using a Twisted's FileBodyProducer, which uses a
Cooperator to schedule calls and can be paused/resumed. Each call takes at
most 65536 bytes from the input.
Both the production input and output are file descriptors, so they can be
applied to a stream of data.
"""
def __init__(self, doc_info, content_fd, secret=None):
# TODO
# This class needs further work to allow for proper streaming.
# RIght now we HAVE TO WAIT until the end of the stream before encoding the
# result. It should be possible to do that just encoding the chunks and
# passing them to a sink, but for that we have to encode the chunks at
# proper alignment (3 byes?) with b64 if armor is defined.
def __init__(self, doc_info, content_fd, secret=None, armor=True,
sink=None):
if not secret:
raise EncryptionDecryptionError('no secret given')
self.doc_id = doc_info.doc_id
self.rev = doc_info.rev
self.armor = armor
self._content_fd = content_fd
self._content_size = self._get_size(content_fd)
self._producer = FileBodyProducer(content_fd, readSize=2**16)
sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
self._aes = AESWriter(sym_key)
self._aes.authenticate(self._make_preamble())
self.sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
self._aes = AESWriter(self.sym_key, _buffer=sink)
self._aes.authenticate(self._encode_preamble())
def _get_size(self, fd):
fd.seek(0, os.SEEK_END)
size = _ceiling(fd.tell())
fd.seek(0)
return size
@property
def iv(self):
......@@ -210,33 +266,94 @@ class BlobEncryptor(object):
callback will be invoked with the resulting ciphertext.
:rtype: twisted.internet.defer.Deferred
"""
# XXX pass a sink to aes?
d = self._producer.startProducing(self._aes)
d.addCallback(lambda _: self._end_crypto_stream())
d.addCallback(lambda _: self._end_crypto_stream_and_encode_result())
return d
def _make_preamble(self):
def _encode_preamble(self):
current_time = int(time.time())
return PACMAN.pack(
preamble = PACMAN.pack(
BLOB_SIGNATURE_MAGIC,
ENC_SCHEME.symkey,
ENC_METHOD.aes_256_gcm,
current_time,
self.iv,
str(self.doc_id),
str(self.rev))
str(self.rev),
self._content_size)
return preamble
def _end_crypto_stream_and_encode_result(self):
# TODO ---- this needs to be refactored to allow PROPER streaming
# We should write the preamble as soon as possible,
# Is it possible to write the AES stream as soon as it is encrypted by
# chunks?
# FIXME also, it needs to be able to encode chunks with base64 if armor
def _end_crypto_stream(self):
preamble, encrypted = self._aes.end()
result = BytesIO()
result.write(
base64.urlsafe_b64encode(preamble))
result.write(' ')
result.write(SEPARATOR)
if self.armor:
result.write(
base64.urlsafe_b64encode(encrypted + self.tag))
else:
result.write(encrypted)
result.write(self.tag)
result.seek(0)
return defer.succeed(result)
class CryptoStreamBodyProducer(FileBodyProducer):
"""
A BodyProducer that gets the tag from the last 16 bytes before closing the
fd.
"""
_tag = None
@property
def tag(self):
# XXX this is a bit tricky. If you call this
# before the end of the stream, you will ruin everything
if not self._tag:
self._writeTag()
return self._tag
def _writeTag(self):
fd = self._inputFile
fd.seek(-16, os.SEEK_END)
self._tag = fd.read(16)
fd.seek(0)
def stopProducing(self):
self._writeTag()
self._inputFile.close()
self._task.stop()
def _writeloop(self, consumer):
"""
Return an iterator which reads one chunk of bytes from the input file
and writes them to the consumer for each time it is iterated.
"""
while True:
bytes = self._inputFile.read(self._readSize)
if not bytes:
self._writeTag()
self._inputFile.close()
break
consumer.write(base64.urlsafe_b64decode(bytes))
yield None
# TODO maybe rename this to just Decryptor, since it will be used by blobs an non
# blobs in soledad.
class BlobDecryptor(object):
"""
Decrypts an encrypted blob associated with a given Document.
......@@ -244,45 +361,71 @@ class BlobDecryptor(object):
Will raise an exception if the blob doesn't have the expected structure, or
if the GCM tag doesn't verify.
"""
# TODO enable the ascii armor = False
def __init__(self, doc_info, ciphertext_fd, result=None,
secret=None):
secret=None, armor=True, start_stream=True):
if not secret:
raise EncryptionDecryptionError('no secret given')
if armor is False:
raise NotImplementedError
self.doc_id = doc_info.doc_id
self.rev = doc_info.rev
self.fd = ciphertext_fd
self.armor = armor
self._producer = None
self.size = None
ciphertext_fd, preamble, iv = self._consume_preamble(ciphertext_fd)
preamble, iv = self._consume_preamble()
assert preamble
assert iv
self.result = result or BytesIO()
sym_key = _get_sym_key_for_doc(doc_info.doc_id, secret)
self._aes = AESWriter(sym_key, iv, self.result, tag=self.tag)
self._aes = AESWriter(sym_key, iv, self.result, tag=None)
self._aes.authenticate(preamble)
if start_stream:
self._start_stream()
self._producer = FileBodyProducer(ciphertext_fd, readSize=2**16)
def _start_stream(self):
self._producer = CryptoStreamBodyProducer(self.fd, readSize=2**16)
self._producer.armor = self.armor
def _consume_preamble(self, ciphertext_fd):
ciphertext_fd.seek(0)
def _consume_preamble(self):
self.fd.seek(0)
try:
preamble, ciphertext = _split(ciphertext_fd.getvalue())
self.tag, ciphertext = ciphertext[-16:], ciphertext[:-16]
except (TypeError, binascii.Error):
raise InvalidBlob
ciphertext_fd.close()
parts = self.fd.getvalue().split()
encoded_preamble = parts[0]
preamble = base64.urlsafe_b64decode(encoded_preamble)
if len(preamble) != PACMAN.size:
except (TypeError, ValueError) as exc:
raise InvalidBlob
try:
unpacked_data = PACMAN.unpack(preamble)
if len(preamble) == LEGACY_PACMAN.size:
warnings.warn("Decrypting a legacy document without size. " +
"This will be deprecated in 0.12. Doc was: " +
"doc_id: %s rev: %s" % (self.doc_id, self.rev),
Warning)
unpacked_data = LEGACY_PACMAN.unpack(preamble)
magic, sch, meth, ts, iv, doc_id, rev = unpacked_data
except struct.error:
raise InvalidBlob
elif len(preamble) == PACMAN.size:
unpacked_data = PACMAN.unpack(preamble)
magic, sch, meth, ts, iv, doc_id, rev, doc_size = unpacked_data
self.size = doc_size
else:
raise InvalidBlob("Unexpected preamble size %d", len(preamble))
except struct.error, e:
raise InvalidBlob(e)
if magic != BLOB_SIGNATURE_MAGIC:
raise InvalidBlob
# TODO check timestamp
# TODO check timestamp. Just as a sanity check, but for instance
# we can refuse to process something that is in the future or
# too far in the past (1984 would be nice, hehe)
if sch != ENC_SCHEME.symkey:
raise InvalidBlob('invalid scheme')
if meth != ENC_METHOD.aes_256_gcm:
......@@ -290,14 +433,26 @@ class BlobDecryptor(object):
if rev != self.rev:
raise InvalidBlob('invalid revision')
if doc_id != self.doc_id:
raise InvalidBlob('invalid revision')
return BytesIO(ciphertext), preamble, iv
raise InvalidBlob('invalid doc id')
self.fd.seek(0)
tail = ''.join(parts[1:])
self.fd.write(tail)
self.fd.seek(len(tail))
self.fd.truncate()
self.fd.seek(0)
return preamble, iv
def _end_stream(self):
try:
return self._aes.end()[1]
self._aes.end()[1]
except InvalidTag:
raise InvalidBlob('Invalid Tag. Blob authentication failed.')
fd = self.result
fd.seek(-16, os.SEEK_END)
fd.truncate()
fd.seek(0)
return self.result
def decrypt(self):
"""
......@@ -307,21 +462,46 @@ class BlobDecryptor(object):
callback will be invoked with the resulting ciphertext.
:rtype: twisted.internet.defer.Deferred
"""
d = self._producer.startProducing(self._aes)
d = self.startProducing()
d.addCallback(lambda _: self._end_stream())
return d
def startProducing(self):
if not self._producer:
self._start_stream()
return self._producer.startProducing(self._aes)
def endStream(self):
self._end_stream()
def write(self, data):
self._aes.write(data)
def close(self):
result = self._aes.end()
return result
@implementer(interfaces.IConsumer)
class AESWriter(object):
"""
A Twisted's Consumer implementation that takes an input file descriptor and
applies AES-256 cipher in GCM mode.
It is used both for encryption and decryption of a stream, depending of the
value of the tag parameter. If you pass a tag, it will operate in
decryption mode, authenticating the preamble. If no tag is passed,
encryption mode is assumed.
"""
implements(interfaces.IConsumer)
def __init__(self, key, iv=None, _buffer=None, tag=None, mode=modes.GCM):
if len(key) != 32:
raise EncryptionDecryptionError('key is not 256 bits')
if tag is not None:
# if tag, we're decrypting
assert iv is not None
self.iv = iv or os.urandom(16)
self.buffer = _buffer or BytesIO()
cipher = _get_aes_cipher(key, self.iv, tag, mode)
......@@ -355,7 +535,8 @@ def is_symmetrically_encrypted(content):
:rtype: bool
"""
return content and content[:13] == '{"raw": "EzcB'
sym_signature = '{"raw": "EzcB'
return content and content.startswith(sym_signature)
# utils
......@@ -375,12 +556,20 @@ def _get_aes_cipher(key, iv, tag, mode=modes.GCM):
return Cipher(algorithms.AES(key), mode, backend=CRYPTO_BACKEND)
def _split(base64_raw_payload):
return imap(base64.urlsafe_b64decode, re.split(' ', base64_raw_payload))
def _mode_by_method(method):
if method == ENC_METHOD.aes_256_gcm:
return modes.GCM
else:
return modes.CTR
def _ceiling(size):
"""
Some simplistic ceiling scheme that uses powers of 2.
We report everything below 4096 bytes as that minimum threshold.
See #8759 for research pending for less simplistic/aggresive strategies.
"""
for i in xrange(12, 31):
step = 2 ** i
if size < step:
return step
0.x.x -
0.9.3 -
+++++++++++++++++++++++++++++++
Please add lines to this file, they will be moved to the CHANGELOG.rst during
......@@ -10,6 +10,10 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff.
Features
~~~~~~~~
- Refactor authentication code to use twisted credential system.
- `#8764 <https://0xacab.org/leap/soledad/issues/8764>`_: Allow unauthenticated
users to retrieve the capabilties banner.
- `#8758 <https://0xacab.org/leap/soledad/issues/8758>`_: Add blob size to the crypto preamble
- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
- New feature without related issue number.
......
#!/bin/bash
######################################################
# Deploy soledad-server from a given remote and branch
# valid remotes are: origin shyba drebs kali
# DO NOT USE IN PRODUCTION OR I'LL SEND NINJAS TO YOUR
# HOUSE!!!!
# (c) LEAP, 2017
######################################################
set -e
REMOTE=$1
BRANCH=$2
if [ "$#" -lt 2 ]; then
echo "USAGE: $0 REMOTE BRANCH"
exit 1
fi
SOLEDADPATH="/usr/lib/python2.7/dist-packages/leap/soledad/server"
REPO="https://0xacab.org/leap/soledad"
LOCALREPO="$HOME/soledad"
SYSTEMDINIT="/lib/systemd/system/soledad-server.service"
apt remove --yes soledad-server
if [ ! -d "$LOCALREPO" ]; then
echo "soledad repo not found, cloning..."
cd $HOME
git clone $REPO
cd $LOCALREPO
git remote add shyba https://0xacab.org/shyba/soledad.git
git remote add drebs https://0xacab.org/drebs/soledad.git
git remote add kali https://0xacab.org/kali/soledad.git
fi
cd $LOCALREPO && git checkout -- . && git fetch $REMOTE && git checkout $REMOTE/$BRANCH
rm -rf $SOLEDADPATH
# copy over some stuff that we'll need -- stolen from debian package
cp -r $LOCALREPO/server/src/leap/soledad/server $SOLEDADPATH
cp $LOCALREPO/server/pkg/soledad-server.service $SYSTEMDINIT
cp $LOCALREPO/server/pkg/create-user-db /usr/bin/
cp $LOCALREPO/server/pkg/soledad-sudoers /etc/sudoers.d/
# Let's append the branch info to the version string! So that nobody is lost
cd $LOCALREPO && echo "__version__ = '"`git describe`"~"`git status | head -n 1 | cut -d' ' -f 4`"'" >> $SOLEDADPATH/__init__.py
# restart the daemon
systemctl --system daemon-reload >/dev/null || true
deb-systemd-invoke restart soledad-server.service >/dev/null || true
tail -n 20 /var/log/syslog
#!/bin/sh
# This script generates a debian package from your current repository tree
# (including modified and unstaged files), using the debian directory from the
# latest debian/platform-X.Y branch.
#
# In order to achieve that, what it does is:
#
# - copy the current repository into a temporary directory.
# - find what is the latest "debian/platform-X.Y" branch.
# - checkout the "debian/" directory from that branch.
# - update the "debian/changelog" file with dummy information.
# - run "debuild -uc -us".
debemail="Leap Automatic Deb Builder <deb@leap.se>"
scriptdir=$(dirname "${0}")
gitroot=$(git -C "${scriptdir}" rev-parse --show-toplevel)
deb_branch=$(git -C "${gitroot}" branch | grep "debian/platform" | sort | tail -n 1 | xargs)
reponame=$(basename "${gitroot}")
tempdir=$(mktemp -d)
targetdir="${tempdir}/${reponame}"
cp -r "${gitroot}" "${tempdir}/${reponame}"
git -C "${targetdir}" checkout "${deb_branch}" -- debian
(cd "${targetdir}" && DEBEMAIL="${debemail}" dch -b "Automatic build.")
(cd "${targetdir}" && debuild -uc -us)
echo "****************************************"
echo "Packages can be found in: ${tempdir}"
ls "${tempdir}"
echo "****************************************"
#!/bin/sh
# This script generates Soledad Debian packages.
#
# When invoking this script, you should pass a git repository URL and the name
# of the branch that contains the code you wish to build the packages from.
#
# The script will clone the given branch from the given repo, as well as the
# main Soledad repo in github which contains the most up-to-date debian
# branch. It will then merge the desired branch into the debian branch and
# build the packages.
if [ $# -ne 2 ]; then
echo "Usage: ${0} <url> <branch>"
exit 1
fi
SOLEDAD_MAIN_REPO=git://github.com/leapcode/soledad.git
url=$1
branch=$2
workdir=`mktemp -d`
git clone -b ${branch} ${url} ${workdir}/soledad
export GIT_DIR=${workdir}/soledad/.git
export GIT_WORK_TREE=${workdir}/soledad
git remote add leapcode ${SOLEDAD_MAIN_REPO}
git fetch leapcode
git checkout -b debian/experimental leapcode/debian/experimental
git merge --no-edit ${branch}
(cd ${workdir}/soledad && debuild -uc -us)
echo "Packages generated in ${workdir}"
......@@ -22,7 +22,7 @@ import argparse
from leap.soledad.common.couch import CouchDatabase
from leap.soledad.common.couch.state import is_db_name_valid
from leap.soledad.common.couch import list_users_dbs
from leap.soledad.server.config import load_configuration
from leap.soledad.server._config import get_config
BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False)
......
Cmnd_Alias SOLEDAD_CREATE_DB = /usr/bin/create-user-db
soledad ALL=(soledad-admin) NOPASSWD: SOLEDAD_CREATE_DB
......@@ -14,21 +14,183 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Blobs Server implementation.
This is a very simplistic implementation for the time being.
Clients should be able to opt-in util the feature is complete.
A more performant BlobsBackend can (and should) be implemented for production
environments.
"""
import commands
import os
from twisted.web import static
from twisted.web import resource
from twisted.web.client import FileBodyProducer
from twisted.web.server import NOT_DONE_YET
from zope.interface import Interface, implementer
__all__ = ['BlobsResource']
# TODO some error handling needed
# [ ] make path configurable
# [ ] sanitize path
# for the future:
# [ ] isolate user avatar in a safer way
# [ ] catch timeout in the server (and delete incomplete upload)
# p [ chunking (should we do it on the client or on the server?)
class BlobAlreadyExists(Exception):
pass
__all__ = ['blobs_resource']
class IBlobsBackend(Interface):
"""
An interface for a BlobsBackend.
"""
def read_blob(user, blob_id, request):
"""
Read blob with a given blob_id, and write it to the passed request.
:returns: a deferred that fires upon finishing.
"""
def write_blob(user, blob_id, request):
"""
Write blob to the storage, reading it from the passed request.
:returns: a deferred that fires upon finishing.
"""
# other stuff for the API
def delete_blob(user, blob_id):
pass
def get_blob_size(user, blob_id):
pass
def get_total_storage(user):
pass
@implementer(IBlobsBackend)
class FilesystemBlobsBackend(object):
path = '/tmp/blobs/'
quota = 200 * 1024 # in KB
def read_blob(self, user, blob_id, request):
print "USER", user
print "BLOB_ID", blob_id
path = self._get_path(user, blob_id)
print "READ FROM", path
_file = static.File(path, defaultType='application/octet-stream')
return _file.render_GET(request)
def write_blob(self, user, blob_id, request):
path = self._get_path(user, blob_id)
if os.path.isfile(path):
# XXX return some 5xx code
raise BlobAlreadyExists()
used = self.get_total_storage(user)
if used > self.quota:
print "Error 507: Quota exceeded for user:", user
request.setResponseCode(507)
request.write('Quota Exceeded!')
request.finish()
return NOT_DONE_YET
try:
os.makedirs(os.path.split(path)[0])
except:
pass
print "WRITE TO", path
fbp = FileBodyProducer(request.content)
d = fbp.startProducing(open(path, 'wb'))
d.addCallback(lambda _: request.finish())
return NOT_DONE_YET
def get_total_storage(self, user):
return self._get_disk_usage(os.path.join(self.path, user))
def delete_blob(user, blob_id):
raise NotImplementedError
def get_blob_size(user, blob_id):
raise NotImplementedError
def _get_disk_usage(self, start_path):
assert os.path.isdir(start_path)
cmd = 'du -c %s | tail -n 1' % start_path
size = commands.getoutput(cmd).split()[0]
return int(size)
def _get_path(self, user, blob_id):
parts = [user]
parts += [blob_id[0], blob_id[0:3], blob_id[0:6]]
parts += [blob_id]
return os.path.join(self.path, *parts)
class BlobsResource(resource.Resource):
isLeaf = True
# Allowed factory classes are defined here
blobsFactoryClass = FilesystemBlobsBackend
def __init__(self):
# TODO pass the backend as configurable option
"""
__init__(self, backend, opts={})
factorykls = getattr(self, backend + 'Class')(**opts)
self._handler = kls()
"""
self._handler = self.blobsFactoryClass()
assert IBlobsBackend.providedBy(self._handler)
# TODO double check credentials, we can have then
# under request.
def render_GET(self, request):
return 'blobs is not implemented yet!'
print "GETTING", request.path
user, blob_id = self._split_path(request.path)
return self._handler.read_blob(user, blob_id, request)
def render_PUT(self, request):
user, blob_id = self._split_path(request.path)
return self._handler.write_blob(user, blob_id, request)
def _split_path(self, blob_id):
# FIXME catch errors here
parts = blob_id.split('/')
_, user, blobname = parts
return user, blobname
if __name__ == '__main__':
# A dummy blob server
# curl -X PUT --data-binary @/tmp/book.pdf localhost:9000/user/someid
# curl -X GET -o /dev/null localhost:9000/user/somerandomstring
from twisted.web.server import Site
from twisted.internet import reactor
# XXX pass the path here
root = BlobsResource()
# I picture somethink like
# BlobsResource(backend="filesystem", backend_opts={'path': '/tmp/blobs'})
blobs_resource = BlobsResource()
factory = Site(root)
reactor.listenTCP(9000, factory)
reactor.run()
......@@ -19,12 +19,35 @@ A twisted resource that serves the Soledad Server.
"""
from twisted.web.resource import Resource
from ._blobs import blobs_resource
from ._blobs import BlobsResource
from ._server_info import ServerInfo
from ._wsgi import get_sync_resource
__all__ = ['SoledadResource']
__all__ = ['SoledadResource', 'SoledadAnonResource']
class _Robots(Resource):
def render_GET(self, request):
return (
'User-agent: *\n'
'Disallow: /\n'
'# you are not a robot, are you???')
class SoledadAnonResource(Resource):
"""
The parts of Soledad Server that unauthenticated users can see.
This is nice because this means that a non-authenticated user will get 404
for anything that is not in this minimal resource tree.
"""
def __init__(self, enable_blobs=False):
Resource.__init__(self)
server_info = ServerInfo(enable_blobs)
self.putChild('', server_info)
self.putChild('robots.txt', _Robots())
class SoledadResource(Resource):
......@@ -51,7 +74,7 @@ class SoledadResource(Resource):
# requests to /blobs will serve blobs if enabled
if enable_blobs:
self.putChild('blobs', blobs_resource)
self.putChild('blobs', BlobsResource())
# other requests are routed to legacy sync resource
self._sync_resource = get_sync_resource(sync_pool)
......
......@@ -21,6 +21,8 @@ import json
from twisted.web.resource import Resource
from leap.soledad.server import __version__
__all__ = ['ServerInfo']
......@@ -35,6 +37,7 @@ class ServerInfo(Resource):
def __init__(self, blobs_enabled):
self._info = {
"blobs": blobs_enabled,
"version": __version__
}
def render_GET(self, request):
......
......@@ -26,19 +26,25 @@ from zope.interface import implementer
from twisted.cred import error
from twisted.cred.checkers import ICredentialsChecker
from twisted.cred.credentials import IUsernamePassword
from twisted.cred.credentials import IAnonymous
from twisted.cred.credentials import Anonymous
from twisted.cred.credentials import UsernamePassword
from twisted.cred.portal import IRealm
from twisted.cred.portal import Portal
from twisted.internet import defer
from twisted.logger import Logger
from twisted.web.iweb import ICredentialFactory
from twisted.web.resource import IResource
from leap.soledad.common.couch import couch_server
from ._resource import SoledadResource
from ._resource import SoledadResource, SoledadAnonResource
from ._config import get_config
log = Logger()
@implementer(IRealm)
class SoledadRealm(object):
......@@ -49,8 +55,22 @@ class SoledadRealm(object):
self._sync_pool = sync_pool
def requestAvatar(self, avatarId, mind, *interfaces):
if IResource in interfaces:
enable_blobs = self._conf['blobs']
# Anonymous access
if IAnonymous.providedBy(avatarId):
resource = SoledadAnonResource(
enable_blobs=enable_blobs)
return (IResource, resource, lambda: None)
# Authenticated users
# TODO review this: #8770 ----------------
# we're creating a Resource tree
# for each request, for every user.
# What are the perf implications of this??
if IResource in interfaces:
resource = SoledadResource(
enable_blobs=enable_blobs,
sync_pool=self._sync_pool)
......@@ -61,7 +81,7 @@ class SoledadRealm(object):
@implementer(ICredentialsChecker)
class TokenChecker(object):
credentialInterfaces = [IUsernamePassword]
credentialInterfaces = [IUsernamePassword, IAnonymous]
TOKENS_DB_PREFIX = "tokens_"
TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds
......@@ -97,6 +117,9 @@ class TokenChecker(object):
return db
def requestAvatarId(self, credentials):
if IAnonymous.providedBy(credentials):
return defer.succeed(Anonymous())
uuid = credentials.username
token = credentials.password
......
......@@ -19,6 +19,7 @@ Twisted resource containing an authenticated Soledad session.
"""
from zope.interface import implementer
from twisted.cred.credentials import Anonymous
from twisted.cred import error
from twisted.python import log
from twisted.web import util
......@@ -59,6 +60,8 @@ class SoledadSession(HTTPAuthSessionWrapper):
self._mapper = URLMapper()
self._portal = portal
self._credentialFactory = credentialFactory
# expected by the contract of the parent class
self._credentialFactories = [credentialFactory]
def _matchPath(self, request):
match = self._mapper.match(request.path, request.method)
......@@ -80,7 +83,7 @@ class SoledadSession(HTTPAuthSessionWrapper):
# get authorization header or fail
header = request.getHeader(b'authorization')
if not header:
return UnauthorizedResource()
return util.DeferredResource(self._login(Anonymous()))
# parse the authorization header
auth_data = self._parseHeader(header)
......@@ -93,6 +96,8 @@ class SoledadSession(HTTPAuthSessionWrapper):
except error.LoginFailed:
return UnauthorizedResource()
except:
# If you port this to the newer log facility, be aware that
# the tests rely on the error to be logged.
log.err(None, "Unexpected failure from credentials factory")
return ErrorPage(500, None, None)
......
......@@ -53,6 +53,7 @@ class URLMapper(object):
URL path | Authorized actions
--------------------------------------------------
/ | GET
/robots.txt | GET
/shared-db | GET
/shared-db/docs | -
/shared-db/doc/{any_id} | GET, PUT, DELETE
......@@ -64,6 +65,8 @@ class URLMapper(object):
"""
# auth info for global resource
self._connect('/', ['GET'])
# robots
self._connect('/robots.txt', ['GET'])
# auth info for shared-db database resource
self._connect('/%s' % SHARED_DB_NAME, ['GET'])
# auth info for shared-db doc resource
......
......@@ -110,7 +110,7 @@ class BlobTestCase(unittest.TestCase):
assert len(preamble) == _crypto.PACMAN.size
unpacked_data = _crypto.PACMAN.unpack(preamble)
magic, sch, meth, ts, iv, doc_id, rev = unpacked_data
magic, sch, meth, ts, iv, doc_id, rev, _ = unpacked_data
assert magic == _crypto.BLOB_SIGNATURE_MAGIC
assert sch == 1
assert meth == _crypto.ENC_METHOD.aes_256_gcm
......@@ -235,10 +235,74 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
_crypto.decrypt_sym(cyphertext, wrongkey, iv)
def _aes_encrypt(key, iv, data):
class PreambleTestCase(unittest.TestCase):
class doc_info:
doc_id = 'D-deadbeef'
rev = '397932e0c77f45fcb7c3732930e7e9b2:1'
def setUp(self):
self.cleartext = BytesIO(snowden1)
self.blob = _crypto.BlobEncryptor(
self.doc_info, self.cleartext,
secret='A' * 96)
def test_preamble_starts_with_magic_signature(self):
preamble = self.blob._encode_preamble()
assert preamble.startswith(_crypto.BLOB_SIGNATURE_MAGIC)
def test_preamble_has_cipher_metadata(self):
preamble = self.blob._encode_preamble()
unpacked = _crypto.PACMAN.unpack(preamble)
encryption_scheme, encryption_method = unpacked[1:3]
assert encryption_scheme in _crypto.ENC_SCHEME
assert encryption_method in _crypto.ENC_METHOD
assert unpacked[4] == self.blob.iv
def test_preamble_has_document_sync_metadata(self):
preamble = self.blob._encode_preamble()
unpacked = _crypto.PACMAN.unpack(preamble)
doc_id, doc_rev = unpacked[5:7]
assert doc_id == self.doc_info.doc_id
assert doc_rev == self.doc_info.rev
def test_preamble_has_document_size(self):
preamble = self.blob._encode_preamble()
unpacked = _crypto.PACMAN.unpack(preamble)
size = unpacked[7]
assert size == len(snowden1)
@defer.inlineCallbacks
def test_preamble_can_come_without_size(self):
# XXX: This test case is here only to test backwards compatibility!
preamble = self.blob._encode_preamble()
# repack preamble using legacy format, without doc size
unpacked = _crypto.PACMAN.unpack(preamble)
preamble_without_size = _crypto.LEGACY_PACMAN.pack(*unpacked[0:7])
# encrypt it manually for custom tag
ciphertext, tag = _aes_encrypt(self.blob.sym_key, self.blob.iv,
self.cleartext.getvalue(),
aead=preamble_without_size)
ciphertext = ciphertext + tag
# encode it
ciphertext = base64.urlsafe_b64encode(ciphertext)
preamble_without_size = base64.urlsafe_b64encode(preamble_without_size)
# decrypt it
ciphertext = preamble_without_size + ' ' + ciphertext
cleartext = yield _crypto.BlobDecryptor(
self.doc_info, BytesIO(ciphertext),
secret='A' * 96).decrypt()
assert cleartext == self.cleartext.getvalue()
warnings = self.flushWarnings()
assert len(warnings) == 1
assert 'legacy document without size' in warnings[0]['message']
def _aes_encrypt(key, iv, data, aead=''):
backend = default_backend()
cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend)
encryptor = cipher.encryptor()
if aead:
encryptor.authenticate_additional_data(aead)
return encryptor.update(data) + encryptor.finalize(), encryptor.tag
......
......@@ -31,9 +31,13 @@ class ServerInfoTestCase(unittest.TestCase):
def test_blobs_enabled(self):
resource = ServerInfo(True)
response = resource.render(DummyRequest(['']))
self.assertEquals(json.loads(response), {'blobs': True})
_info = json.loads(response)
self.assertEquals(_info['blobs'], True)
self.assertTrue(isinstance(_info['version'], basestring))
def test_blobs_disabled(self):
resource = ServerInfo(False)
response = resource.render(DummyRequest(['']))
self.assertEquals(json.loads(response), {'blobs': False})
_info = json.loads(response)
self.assertEquals(_info['blobs'], False)
self.assertTrue(isinstance(_info['version'], basestring))
......@@ -156,6 +156,7 @@ class SoledadSessionTestCase(unittest.TestCase):
return {}
def decode(self, response, request):
print "decode raised"
raise UnexpectedException()
self.wrapper._credentialFactory = BadFactory()
......@@ -164,7 +165,8 @@ class SoledadSessionTestCase(unittest.TestCase):
child = getChildForRequest(self.wrapper, request)
request.render(child)
self.assertEqual(request.responseCode, 500)
self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1)
errors = self.flushLoggedErrors(UnexpectedException)
self.assertEqual(len(errors), 1)
def test_unexpectedLoginError(self):
class UnexpectedException(Exception):
......