[refactor] improve secrets generation and storage code (96c2aa02) · Commits · leap / soledad

client/src/leap/soledad/client/_secrets/init.py

0 → 100644

+132 −0

Original line number	Diff line number	Diff line
		# -- coding: utf-8 --
		# _secrets/__init__.py
		# Copyright (C) 2016 LEAP
		#
		# This program is free software: you can redistribute it and/or modify
		# it under the terms of the GNU General Public License as published by
		# the Free Software Foundation, either version 3 of the License, or
		# (at your option) any later version.
		#
		# This program is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		# GNU General Public License for more details.
		#
		# You should have received a copy of the GNU General Public License
		# along with this program. If not, see <http://www.gnu.org/licenses/>.

		import os

		from collections import namedtuple

		from leap.soledad.common.log import getLogger

		from leap.soledad.client._secrets.storage import SecretsStorage
		from leap.soledad.client._secrets.crypto import SecretsCrypto
		from leap.soledad.client._secrets.util import emit


		logger = getLogger(__name__)


		SecretLength = namedtuple('SecretLength', 'name length')


		class Secrets(object):

		lengths = {
		'remote': 512,
		'salt': 64,
		'local': 448,
		}

		def __init__(self, uuid, passphrase, url, local_path, creds, userid,
		shared_db=None):
		self._passphrase = passphrase
		self._secrets = {}
		self._user_data = {'uuid': uuid, 'userid': userid}
		self.crypto = SecretsCrypto(self.get_passphrase)
		self.storage = SecretsStorage(
		uuid, self.get_passphrase, url, local_path, creds, userid,
		shared_db=shared_db)
		self._bootstrap()

		#
		# bootstrap
		#

		def _bootstrap(self):
		force_storage = False

		# attempt to load secrets from local storage
		encrypted = self.storage.load_local()

		# if not found, attempt to load secrets from remote storage
		if not encrypted:
		encrypted = self.storage.load_remote()

		if not encrypted:
		# if not found, generate new secrets
		secrets = self._generate()
		encrypted = self.crypto.encrypt(secrets)
		force_storage = True
		else:
		# decrypt secrets found either in local or remote storage
		secrets = self.crypto.decrypt(encrypted)

		self._secrets = secrets

		if encrypted['version'] < self.crypto.VERSION or force_storage:
		self.storage.save_local(encrypted)
		self.storage.save_remote(encrypted)

		#
		# generation
		#

		@emit('creating')
		def _generate(self):
		logger.info("generating new set of secrets...")
		secrets = {}
		for name, length in self.lengths.iteritems():
		secret = os.urandom(length)
		secrets[name] = secret
		logger.info("new set of secrets successfully generated")
		return secrets

		#
		# crypto
		#

		def _encrypt(self):
		# encrypt secrets
		secrets = self._secrets
		encrypted = self.crypto.encrypt(secrets)
		# create the recovery document
		data = {'secret': encrypted, 'version': 2}
		return data

		def get_passphrase(self):
		return self._passphrase.encode('utf-8')

		@property
		def passphrase(self):
		return self.get_passphrase()

		def change_passphrase(self, new_passphrase):
		self._passphrase = new_passphrase
		encrypted = self.crypto.encrypt(self._secrets)
		self.storage.save_local(encrypted)
		self.storage.save_remote(encrypted)

		@property
		def remote(self):
		return self._secrets.get('remote')

		@property
		def salt(self):
		return self._secrets.get('salt')

		@property
		def local(self):
		return self._secrets.get('local')

client/src/leap/soledad/client/_secrets/crypto.py

0 → 100644

+123 −0

Original line number	Diff line number	Diff line
		# -- coding: utf-8 --
		# _secrets/crypto.py
		# Copyright (C) 2016 LEAP
		#
		# This program is free software: you can redistribute it and/or modify
		# it under the terms of the GNU General Public License as published by
		# the Free Software Foundation, either version 3 of the License, or
		# (at your option) any later version.
		#
		# This program is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		# GNU General Public License for more details.
		#
		# You should have received a copy of the GNU General Public License
		# along with this program. If not, see <http://www.gnu.org/licenses/>.

		import binascii
		import json
		import os
		import scrypt

		from leap.soledad.common import soledad_assert
		from leap.soledad.common.log import getLogger

		from leap.soledad.client._crypto import encrypt_sym, decrypt_sym, ENC_METHOD
		from leap.soledad.client._secrets.util import SecretsError


		logger = getLogger(__name__)


		class SecretsCrypto(object):

		VERSION = 2

		def __init__(self, get_pass):
		self._get_pass = get_pass

		def _get_key(self, salt):
		key = scrypt.hash(self._get_pass(), salt, buflen=32)
		return key

		#
		# encryption
		#

		def encrypt(self, secrets):
		encoded = {}
		for name, value in secrets.iteritems():
		encoded[name] = binascii.b2a_base64(value)
		plaintext = json.dumps(encoded)
		salt = os.urandom(64) # TODO: get salt length from somewhere else
		key = self._get_key(salt)
		iv, ciphertext = encrypt_sym(plaintext, key,
		method=ENC_METHOD.aes_256_gcm)
		encrypted = {
		'version': self.VERSION,
		'kdf': 'scrypt',
		'kdf_salt': binascii.b2a_base64(salt),
		'kdf_length': len(key),
		'cipher': 'aes_256_gcm',
		'length': len(plaintext),
		'iv': str(iv),
		'secrets': binascii.b2a_base64(ciphertext),
		}
		return encrypted

		#
		# decryption
		#

		def decrypt(self, data):
		version = data.get('version')
		method = getattr(self, '_decrypt_v%d' % version)
		try:
		return method(data)
		except Exception as e:
		logger.error('error decrypting secrets: %r' % e)
		raise SecretsError(e)

		def _decrypt_v1(self, data):
		secret_id = data['active_secret']
		encrypted = data['storage_secrets'][secret_id]
		soledad_assert(encrypted['cipher'] == 'aes256')

		salt = binascii.a2b_base64(encrypted['kdf_salt'])
		key = self._get_key(salt)
		separator = ':'
		iv, ciphertext = encrypted['secret'].split(separator, 1)
		ciphertext = binascii.a2b_base64(ciphertext)
		plaintext = self._decrypt(
		key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_ctr)
		secrets = {
		'remote': plaintext[0:512],
		'salt': plaintext[512:576],
		'local': plaintext[576:1024],
		}
		return secrets

		def _decrypt_v2(self, encrypted):
		soledad_assert(encrypted['cipher'] == 'aes_256_gcm')

		salt = binascii.a2b_base64(encrypted['kdf_salt'])
		key = self._get_key(salt)
		iv = encrypted['iv']
		ciphertext = binascii.a2b_base64(encrypted['secrets'])
		plaintext = self._decrypt(
		key, iv, ciphertext, encrypted, ENC_METHOD.aes_256_gcm)
		encoded = json.loads(plaintext)
		secrets = {}
		for name, value in encoded.iteritems():
		secrets[name] = binascii.a2b_base64(value)
		return secrets

		def _decrypt(self, key, iv, ciphertext, encrypted, method):
		# assert some properties of the stored secret
		soledad_assert(encrypted['kdf'] == 'scrypt')
		soledad_assert(encrypted['kdf_length'] == len(key))
		# decrypt
		plaintext = decrypt_sym(ciphertext, key, iv, method)
		soledad_assert(encrypted['length'] == len(plaintext))
		return plaintext

client/src/leap/soledad/client/_secrets/storage.py

0 → 100644

+124 −0

Original line number	Diff line number	Diff line
		# -- coding: utf-8 --
		# _secrets/storage.py
		# Copyright (C) 2016 LEAP
		#
		# This program is free software: you can redistribute it and/or modify
		# it under the terms of the GNU General Public License as published by
		# the Free Software Foundation, either version 3 of the License, or
		# (at your option) any later version.
		#
		# This program is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		# GNU General Public License for more details.
		#
		# You should have received a copy of the GNU General Public License
		# along with this program. If not, see <http://www.gnu.org/licenses/>.

		import json
		import urlparse

		from hashlib import sha256

		from leap.soledad.common import SHARED_DB_NAME
		from leap.soledad.common.log import getLogger

		from leap.soledad.common.document import SoledadDocument
		from leap.soledad.client.shared_db import SoledadSharedDatabase
		from leap.soledad.client._secrets.util import emit


		logger = getLogger(__name__)


		class SecretsStorage(object):

		def __init__(self, uuid, get_pass, url, local_path, creds, userid,
		shared_db=None):
		self._uuid = uuid
		self._get_pass = get_pass
		self._local_path = local_path
		self._userid = userid

		self._shared_db = shared_db or self._init_shared_db(url, creds)
		self.__remote_doc = None

		#
		# properties
		#

		@property
		def _user_data(self):
		return {'uuid': self._uuid, 'userid': self._userid}

		#
		# local storage
		#

		def load_local(self):
		logger.info("trying to load secrets from disk: %s" % self._local_path)
		try:
		with open(self._local_path, 'r') as f:
		encrypted = json.loads(f.read())
		logger.info("secrets loaded successfully from disk")
		return encrypted
		except IOError:
		logger.warn("secrets not found in disk")
		return None

		def save_local(self, encrypted):
		json_data = json.dumps(encrypted)
		with open(self._local_path, 'w') as f:
		f.write(json_data)

		#
		# remote storage
		#

		def _init_shared_db(self, url, creds):
		url = urlparse.urljoin(url, SHARED_DB_NAME)
		db = SoledadSharedDatabase.open_database(
		url, self._uuid, creds=creds)
		self._shared_db = db

		def _remote_doc_id(self):
		passphrase = self._get_pass()
		text = '%s%s' % (passphrase, self._uuid)
		digest = sha256(text).hexdigest()
		return digest

		@property
		def _remote_doc(self):
		if not self.__remote_doc and self._shared_db:
		doc = self._get_remote_doc()
		self.__remote_doc = doc
		return self.__remote_doc

		@emit('downloading')
		def _get_remote_doc(self):
		logger.info('trying to load secrets from server...')
		doc = self._shared_db.get_doc(self._remote_doc_id())
		if doc:
		logger.info('secrets loaded successfully from server')
		else:
		logger.warn('secrets not found in server')
		return doc

		def load_remote(self):
		doc = self._remote_doc
		if not doc:
		return None
		encrypted = doc.content
		return encrypted

		@emit('uploading')
		def save_remote(self, encrypted):
		doc = self._remote_doc
		if not doc:
		doc = SoledadDocument(doc_id=self._remote_doc_id())
		doc.content = encrypted
		db = self._shared_db
		if not db:
		logger.warn('no shared db found')
		return
		db.put_doc(doc)

client/src/leap/soledad/client/_secrets/util.py

0 → 100644

+46 −0

Original line number	Diff line number	Diff line
		# -- coding:utf-8 --
		# _secrets/util.py
		# Copyright (C) 2016 LEAP
		#
		# This program is free software: you can redistribute it and/or modify
		# it under the terms of the GNU General Public License as published by
		# the Free Software Foundation, either version 3 of the License, or
		# (at your option) any later version.
		#
		# This program is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		# GNU General Public License for more details.
		#
		# You should have received a copy of the GNU General Public License
		# along with this program. If not, see <http://www.gnu.org/licenses/>.


		from leap.soledad.client import events


		class SecretsError(Exception):
		pass


		def emit(verb):
		def _decorator(method):
		def _decorated(self, args, *kwargs):

		# emit starting event
		user_data = self._user_data
		name = 'SOLEDAD_' + verb.upper() + '_KEYS'
		event = getattr(events, name)
		events.emit_async(event, user_data)

		# run the method
		result = method(self, args, *kwargs)

		# emit a finished event
		name = 'SOLEDAD_DONE_' + verb.upper() + '_KEYS'
		event = getattr(events, name)
		events.emit_async(event, user_data)

		return result
		return _decorated
		return _decorator

client/src/leap/soledad/client/api.py

+11 −89

Original line number	Diff line number	Diff line
		@@ -45,7 +45,6 @@ from zope.interface import implements
		from leap.common.config import get_path_prefix
		from leap.common.plugins import collect_plugins

		from leap.soledad.common import SHARED_DB_NAME
		from leap.soledad.common import soledad_assert
		from leap.soledad.common import soledad_assert_type
		from leap.soledad.common.log import getLogger
		@@ -57,8 +56,7 @@ from leap.soledad.client import adbapi
		from leap.soledad.client import events as soledad_events
		from leap.soledad.client import interfaces as soledad_interfaces
		from leap.soledad.client import sqlcipher
		from leap.soledad.client.secrets import SoledadSecrets
		from leap.soledad.client.shared_db import SoledadSharedDatabase
		from leap.soledad.client._secrets import Secrets
		from leap.soledad.client._crypto import SoledadCrypto

		logger = getLogger(__name__)
		@@ -130,7 +128,7 @@ class Soledad(object):

		def __init__(self, uuid, passphrase, secrets_path, local_db_path,
		server_url, cert_file, shared_db=None,
		auth_token=None, syncable=True):
		auth_token=None):
		"""
		Initialize configuration, cryptographic keys and dbs.

		@@ -185,8 +183,6 @@ class Soledad(object):
		self._secrets_path = None
		self._dbsyncer = None

		self.shared_db = shared_db

		# configure SSL certificate
		global SOLEDAD_CERT
		SOLEDAD_CERT = cert_file
		@@ -198,20 +194,14 @@ class Soledad(object):

		self._secrets_path = secrets_path

		# Initialize shared recovery database
		self.init_shared_db(server_url, uuid, self._creds, syncable=syncable)

		# The following can raise BootstrapSequenceError, that will be
		# propagated upwards.
		self._init_secrets()
		self._init_secrets(shared_db=shared_db)

		self._crypto = SoledadCrypto(self._secrets.remote_storage_secret)
		self._crypto = SoledadCrypto(self._secrets.remote)

		try:
		# initialize database access, trap any problems so we can shutdown
		# smoothly.
		self._init_u1db_sqlcipher_backend()
		if syncable:
		self._init_u1db_syncer()
		except DatabaseAccessError:
		# oops! something went wrong with backend initialization. We
		@@ -255,14 +245,13 @@ class Soledad(object):
		for path in paths:
		create_path_if_not_exists(path)

		def _init_secrets(self):
		def _init_secrets(self, shared_db=None):
		"""
		Initialize Soledad secrets.
		"""
		self._secrets = SoledadSecrets(
		self.uuid, self._passphrase, self._secrets_path,
		self.shared_db, userid=self.userid)
		self._secrets.bootstrap()
		self._secrets = Secrets(
		self._uuid, self._passphrase, self._server_url, self._secrets_path,
		self._creds, self.userid, shared_db=shared_db)

		def _init_u1db_sqlcipher_backend(self):
		"""
		@@ -279,7 +268,7 @@ class Soledad(object):
		"""
		tohex = binascii.b2a_hex
		# sqlcipher only accepts the hex version
		key = tohex(self._secrets.get_local_storage_key())
		key = tohex(self._secrets.local)

		opts = sqlcipher.SQLCipherOptions(
		self._local_db_path, key,
		@@ -659,21 +648,6 @@ class Soledad(object):
		# ISyncableStorage
		#

		def set_syncable(self, syncable):
		"""
		Toggle the syncable state for this database.

		This can be used to start a database with offline state and switch it
		online afterwards. Or the opposite: stop syncs when connection is lost.

		:param syncable: new status for syncable.
		:type syncable: bool
		"""
		# TODO should check that we've got a token!
		self.shared_db.syncable = syncable
		if syncable and not self._dbsyncer:
		self._init_u1db_syncer()

		def sync(self):
		"""
		Synchronize documents with the server replica.
		@@ -760,13 +734,6 @@ class Soledad(object):
		"""
		return self.sync_lock.locked

		@property
		def syncable(self):
		if self.shared_db:
		return self.shared_db.syncable
		else:
		return False

		def _set_token(self, token):
		"""
		Set the authentication token for remote database access.
		@@ -803,58 +770,13 @@ class Soledad(object):
		# ISecretsStorage
		#

		def init_shared_db(self, server_url, uuid, creds, syncable=True):
		"""
		Initialize the shared database.

		:param server_url: URL of the remote database.
		:type server_url: str
		:param uuid: The user's unique id.
		:type uuid: str
		:param creds: A tuple containing the authentication method and
		credentials.
		:type creds: tuple
		:param syncable:
		If syncable is False, the database will not attempt to sync against
		a remote replica.
		:type syncable: bool
		"""
		# only case this is False is for testing purposes
		if self.shared_db is None:
		shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME)
		self.shared_db = SoledadSharedDatabase.open_database(
		shared_db_url,
		uuid,
		creds=creds,
		syncable=syncable)

		@property
		def storage_secret(self):
		"""
		Return the secret used for local storage encryption.

		:return: The secret used for local storage encryption.
		:rtype: str
		"""
		return self._secrets.storage_secret

		@property
		def remote_storage_secret(self):
		"""
		Return the secret used for encryption of remotely stored data.

		:return: The secret used for remote storage encryption.
		:rtype: str
		"""
		return self._secrets.remote_storage_secret

		@property
		def secrets(self):
		"""
		Return the secrets object.

		:return: The secrets object.
		:rtype: SoledadSecrets
		:rtype: Secrets
		"""
		return self._secrets