diff --git a/CHANGELOG b/CHANGELOG
index 660d4160ae8ef18f9c8510c2d264f1518641cc61..83bb883e0cfc60875805d68d886136bddefa00d5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,13 @@
+0.4.4 Dec 6:
+Client:
+ o Add MAC verirication to the recovery document and
+ soledad.json. Closes #4348.
+Common:
+ o Add unicode conversion to put_doc(). Closes #4095.
+ o Remove tests dependency on nose2. Closes #4258.
+
0.4.3 Nov 15:
+Client:
o Defaults detected encoding to utf-8 to avoid bug if detected
encoding is None. Closes: #4417
o Open db in autocommit mode, to avoid nested transactions problems.
diff --git a/README.rst b/README.rst
index 61a15ee4b67488b296b97d851289d3f38e7d57b0..fb909120c0d6e8eb2381073176858acb8f7b2fbf 100644
--- a/README.rst
+++ b/README.rst
@@ -38,14 +38,8 @@ Tests
Client and server tests are both included in leap.soledad.common. If you want
to run tests in development mode you must do the following::
- cd common
- python setup.py develop
- cd ../client
- python setup.py develop
- cd ../server
- python setup.py develop
- cd ../common
- python setup.py test
+ scripts/develop_mode.sh
+ ./run_tests.sh
Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your
system.
diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index a159d77384217b171a0b945c05c4112f8f8cf087..62f93b3d4e18146efd5daefb88d3049a9f0ec494 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -31,6 +31,7 @@ import os
import socket
import ssl
import urlparse
+import hmac
from hashlib import sha256
@@ -52,6 +53,13 @@ from leap.soledad.common.errors import (
NotLockedError,
AlreadyLockedError,
)
+from leap.soledad.common.crypto import (
+ MacMethods,
+ UnknownMacMethod,
+ WrongMac,
+ MAC_KEY,
+ MAC_METHOD_KEY,
+)
#
# Signaling function
@@ -357,7 +365,12 @@ class Soledad(object):
logger.info(
'Found cryptographic secrets in shared recovery '
'database.')
- self.import_recovery_document(doc.content)
+ _, mac = self.import_recovery_document(doc.content)
+ if mac is False:
+ self.put_secrets_in_shared_db()
+ self._store_secrets() # save new secrets in local file
+ if self._secret_id is None:
+ self._set_secret_id(self._secrets.items()[0][0])
else:
# STAGE 3 - there are no secrets in server also, so
# generate a secret and store it in remote db.
@@ -516,21 +529,6 @@ class Soledad(object):
def _load_secrets(self):
"""
Load storage secrets from local file.
-
- The content of the file has the following format:
-
- {
- "storage_secrets": {
- "<secret_id>": {
- 'kdf': 'scrypt',
- 'kdf_salt': '<b64 repr of salt>'
- 'kdf_length': <key length>
- "cipher": "aes256",
- "length": <secret length>,
- "secret": "<encrypted storage_secret 1>",
- }
- }
- }
"""
# does the file exist in disk?
if not os.path.isfile(self._secrets_path):
@@ -539,7 +537,10 @@ class Soledad(object):
content = None
with open(self._secrets_path, 'r') as f:
content = json.loads(f.read())
- self._secrets = content[self.STORAGE_SECRETS_KEY]
+ _, mac = self.import_recovery_document(content)
+ if mac is False:
+ self._store_secrets()
+ self._put_secrets_in_shared_db()
# choose first secret if no secret_id was given
if self._secret_id is None:
self._set_secret_id(self._secrets.items()[0][0])
@@ -614,28 +615,12 @@ class Soledad(object):
def _store_secrets(self):
"""
- Store a secret in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
-
- The contents of the stored file have the following format:
-
- {
- 'storage_secrets': {
- '<secret_id>': {
- 'kdf': 'scrypt',
- 'kdf_salt': '<salt>'
- 'kdf_length': <len>
- 'cipher': 'aes256',
- 'length': 1024,
- 'secret': '<encrypted storage_secret 1>',
- }
- }
- }
+ Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
"""
- data = {
- self.STORAGE_SECRETS_KEY: self._secrets,
- }
with open(self._secrets_path, 'w') as f:
- f.write(json.dumps(data))
+ f.write(
+ json.dumps(
+ self.export_recovery_document()))
def change_passphrase(self, new_passphrase):
"""
@@ -662,6 +647,7 @@ class Soledad(object):
# get a 256-bit key
key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
iv, ciphertext = self._crypto.encrypt_sym(secret, key)
+ # XXX update all secrets in the dict
self._secrets[self._secret_id] = {
# leap.soledad.crypto submodule uses AES256 for symmetric
# encryption.
@@ -673,9 +659,9 @@ class Soledad(object):
self.SECRET_KEY: '%s%s%s' % (
str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
}
-
- self._store_secrets()
self._passphrase = new_passphrase
+ self._store_secrets()
+ self._put_secrets_in_shared_db()
#
# General crypto utility methods.
@@ -743,7 +729,7 @@ class Soledad(object):
doc = SoledadDocument(
doc_id=self._shared_db_doc_id())
# fill doc with encrypted secrets
- doc.content = self.export_recovery_document(include_uuid=False)
+ doc.content = self.export_recovery_document()
# upload secrets to server
signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
db = self._shared_db
@@ -761,12 +747,20 @@ class Soledad(object):
"""
Update a document in the local encrypted database.
+ ============================== WARNING ==============================
+ This method converts the document's contents to unicode in-place. This
+ meanse that after calling C{put_doc(doc)}, the contents of the
+ document, i.e. C{doc.content}, might be different from before the
+ call.
+ ============================== WARNING ==============================
+
:param doc: the document to update
:type doc: SoledadDocument
:return: the new revision identifier for the document
:rtype: str
"""
+ doc.content = self._convert_to_unicode(doc.content)
return self._db.put_doc(doc)
def delete_doc(self, doc):
@@ -1100,26 +1094,51 @@ class Soledad(object):
# Recovery document export and import methods
#
- def export_recovery_document(self, include_uuid=True):
+ def export_recovery_document(self):
"""
- Export the storage secrets and (optionally) the uuid.
+ Export the storage secrets.
A recovery document has the following structure:
{
- self.STORAGE_SECRET_KEY: <secrets dict>,
- self.UUID_KEY: '<uuid>', # (optional)
+ 'storage_secrets': {
+ '<storage_secret id>': {
+ 'kdf': 'scrypt',
+ 'kdf_salt': '<b64 repr of salt>'
+ 'kdf_length': <key length>
+ 'cipher': 'aes256',
+ 'length': <secret length>,
+ 'secret': '<encrypted storage_secret>',
+ },
+ },
+ 'kdf': 'scrypt',
+ 'kdf_salt': '<b64 repr of salt>',
+ 'kdf_length: <key length>,
+ '_mac_method': 'hmac',
+ '_mac': '<mac>'
}
- :param include_uuid: Should the uuid be included?
- :type include_uuid: bool
+ Note that multiple storage secrets might be stored in one recovery
+ document. This method will also calculate a MAC of a string
+ representation of the secrets dictionary.
:return: The recovery document.
:rtype: dict
"""
- data = {self.STORAGE_SECRETS_KEY: self._secrets}
- if include_uuid:
- data[self.UUID_KEY] = self._uuid
+ # create salt and key for calculating MAC
+ salt = os.urandom(self.SALT_LENGTH)
+ key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+ data = {
+ self.STORAGE_SECRETS_KEY: self._secrets,
+ self.KDF_KEY: self.KDF_SCRYPT,
+ self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+ self.KDF_LENGTH_KEY: len(key),
+ MAC_METHOD_KEY: MacMethods.HMAC,
+ MAC_KEY: hmac.new(
+ key,
+ json.dumps(self._secrets),
+ sha256).hexdigest(),
+ }
return data
def import_recovery_document(self, data):
@@ -1127,27 +1146,49 @@ class Soledad(object):
Import storage secrets for symmetric encryption and uuid (if present)
from a recovery document.
- A recovery document has the following structure:
-
- {
- self.STORAGE_SECRET_KEY: <secrets dict>,
- self.UUID_KEY: '<uuid>', # (optional)
- }
+ Note that this method does not store the imported data on disk. For
+ that, use C{self._store_secrets()}.
:param data: The recovery document.
:type data: dict
- """
- # include new secrets in our secret pool.
+
+ :return: A tuple containing the number of imported secrets and whether
+ there was MAC informationa available for authenticating.
+ :rtype: (int, bool)
+ """
+ soledad_assert(self.STORAGE_SECRETS_KEY in data)
+ # check mac of the recovery document
+ mac_auth = False
+ mac = None
+ if MAC_KEY in data:
+ soledad_assert(data[MAC_KEY] is not None)
+ soledad_assert(MAC_METHOD_KEY in data)
+ soledad_assert(self.KDF_KEY in data)
+ soledad_assert(self.KDF_SALT_KEY in data)
+ soledad_assert(self.KDF_LENGTH_KEY in data)
+ if data[MAC_METHOD_KEY] == MacMethods.HMAC:
+ key = scrypt.hash(
+ self._passphrase_as_string(),
+ binascii.a2b_base64(data[self.KDF_SALT_KEY]),
+ buflen=32)
+ mac = hmac.new(
+ key,
+ json.dumps(data[self.STORAGE_SECRETS_KEY]),
+ sha256).hexdigest()
+ else:
+ raise UnknownMacMethod('Unknown MAC method: %s.' %
+ data[MAC_METHOD_KEY])
+ if mac != data[MAC_KEY]:
+ raise WrongMac('Could not authenticate recovery document\'s '
+ 'contents.')
+ mac_auth = True
+ # include secrets in the secret pool.
+ secrets = 0
for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
if secret_id not in self._secrets:
+ secrets += 1
self._secrets[secret_id] = secret_data
- self._store_secrets() # save new secrets in local file
- # set uuid if present
- if self.UUID_KEY in data:
- self._uuid = data[self.UUID_KEY]
- # choose first secret to use is none is assigned
- if self._secret_id is None:
- self._set_secret_id(data[self.STORAGE_SECRETS_KEY].items()[0][0])
+ return secrets, mac
#
# Setters/getters
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
index 6563988762922b7a357dbac563c37c7136845f97..d8899a97a1aec5b54a34ab288554afa7771fc3f3 100644
--- a/client/src/leap/soledad/client/target.py
+++ b/client/src/leap/soledad/client/target.py
@@ -35,7 +35,10 @@ from u1db.remote.http_target import HTTPSyncTarget
from leap.soledad.common import soledad_assert
from leap.soledad.common.crypto import (
EncryptionSchemes,
+ UnknownEncryptionScheme,
MacMethods,
+ UnknownMacMethod,
+ WrongMac,
ENC_JSON_KEY,
ENC_SCHEME_KEY,
ENC_METHOD_KEY,
@@ -62,27 +65,6 @@ class DocumentNotEncrypted(Exception):
pass
-class UnknownEncryptionScheme(Exception):
- """
- Raised when trying to decrypt from unknown encryption schemes.
- """
- pass
-
-
-class UnknownMacMethod(Exception):
- """
- Raised when trying to authenticate document's content with unknown MAC
- mehtod.
- """
- pass
-
-
-class WrongMac(Exception):
- """
- Raised when failing to authenticate document's contents based on MAC.
- """
-
-
#
# Crypto utilities for a SoledadDocument.
#
diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip
index 6ff52ff968bc7d459154a2a6d0c9b8e58e1a3e2a..9302450c85caac501ebc728e61f1571ed709b71e 100644
--- a/common/pkg/requirements-testing.pip
+++ b/common/pkg/requirements-testing.pip
@@ -1,10 +1,5 @@
mock
-nose2
testscenarios
leap.common
leap.soledad.server
leap.soledad.client
-
-# Under quarantine...
-# I've been able to run all tests with six==1.3 -- kali
-# six==1.1.0 # some tests are incompatible with newer versions of six.
diff --git a/common/pkg/tools/with_venvwrapper.sh b/common/pkg/tools/with_venvwrapper.sh
new file mode 100755
index 0000000000000000000000000000000000000000..b62bc10fca88bb047176fd87a712f7461bddd114
--- /dev/null
+++ b/common/pkg/tools/with_venvwrapper.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#Wraps a command in a virtualenwrapper passed as first argument.
+#Example:
+#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh
+
+wd=`pwd`
+alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"'
+
+source `which virtualenvwrapper.sh`
+echo "Activating virtualenv " $1
+echo "------------------------------------"
+workon $1
+cd $wd
+echo "running version: "
+echo `pyver leap.bitmask`
+echo `pyver leap.soledad.common`
+echo `pyver leap.keymanager`
+$2 $3 $4 $5
diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py
index 2c6bd7a36f820e1d7b748159f7e3095a922effd5..56bb608aed778cf7ebbc1b6d8b8793d4825d5909 100644
--- a/common/src/leap/soledad/common/crypto.py
+++ b/common/src/leap/soledad/common/crypto.py
@@ -35,6 +35,13 @@ class EncryptionSchemes(object):
PUBKEY = 'pubkey'
+class UnknownEncryptionScheme(Exception):
+ """
+ Raised when trying to decrypt from unknown encryption schemes.
+ """
+ pass
+
+
class MacMethods(object):
"""
Representation of MAC methods used to authenticate document's contents.
@@ -43,6 +50,20 @@ class MacMethods(object):
HMAC = 'hmac'
+class UnknownMacMethod(Exception):
+ """
+ Raised when trying to authenticate document's content with unknown MAC
+ mehtod.
+ """
+ pass
+
+
+class WrongMac(Exception):
+ """
+ Raised when failing to authenticate document's contents based on MAC.
+ """
+
+
#
# Crypto utilities for a SoledadDocument.
#
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index db217bb3d01852ed768357fdfa77faa3e9d4754c..af11bc7681bd0795fd5a003f8b0ec0556f02d493 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -40,6 +40,7 @@ from leap.soledad.common.tests import (
KEY_FINGERPRINT,
PRIVATE_KEY,
)
+from leap.soledad.common.crypto import WrongMac, UnknownMacMethod
from leap.soledad.common.tests.u1db_tests import (
simple_doc,
nested_doc,
@@ -88,11 +89,9 @@ class RecoveryDocumentTestCase(BaseSoledadTest):
def test_import_recovery_document(self):
rd = self._soledad.export_recovery_document()
- s = self._soledad_instance(user='anotheruser@leap.se')
+ s = self._soledad_instance()
s.import_recovery_document(rd)
s._set_secret_id(self._soledad._secret_id)
- self.assertEqual(self._soledad._uuid,
- s._uuid, 'Failed setting user uuid.')
self.assertEqual(self._soledad._get_storage_secret(),
s._get_storage_secret(),
'Failed settinng secret for symmetric encryption.')
@@ -164,7 +163,7 @@ class MacAuthTestCase(BaseSoledadTest):
doc.content[target.MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(
- target.WrongMac,
+ WrongMac,
target.decrypt_doc, self._soledad._crypto, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
@@ -182,7 +181,7 @@ class MacAuthTestCase(BaseSoledadTest):
doc.content[target.MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(
- target.UnknownMacMethod,
+ UnknownMacMethod,
target.decrypt_doc, self._soledad._crypto, doc)
diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index 8970a4371c5d9cd9e54dc31e1a516e2cdb0c09bf..035c5ac5e1d1ef5b191235b846fad56970e84ba8 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -33,6 +33,7 @@ from leap.soledad.common.tests import (
)
from leap import soledad
from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.crypto import WrongMac
from leap.soledad.client import Soledad, PassphraseTooShort
from leap.soledad.client.crypto import SoledadCrypto
from leap.soledad.client.shared_db import SoledadSharedDatabase
@@ -119,7 +120,7 @@ class AuxMethodsTestCase(BaseSoledadTest):
sol.change_passphrase(u'654321')
self.assertRaises(
- DatabaseError,
+ WrongMac,
self._soledad_instance, 'leap@leap.se',
passphrase=u'123',
prefix=self.rand_prefix)
@@ -292,7 +293,7 @@ class SoledadSignalingTestCase(BaseSoledadTest):
sol = self._soledad_instance()
# create a document with secrets
doc = SoledadDocument(doc_id=sol._shared_db_doc_id())
- doc.content = sol.export_recovery_document(include_uuid=False)
+ doc.content = sol.export_recovery_document()
class Stage2MockSharedDB(object):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README
index 605f01faeecf3d1af4be84b881bca25555cc9cb6..d543f25090426cc8295ceda10e24e6641e638012 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/README
+++ b/common/src/leap/soledad/common/tests/u1db_tests/README
@@ -12,7 +12,6 @@ Dependencies
u1db tests depend on the following python packages:
- nose2
unittest2
mercurial
hgtools
@@ -25,10 +24,3 @@ u1db tests depend on the following python packages:
routes
simplejson
cython
-
-Running tests
--------------
-
-Use nose2 to run tests:
-
- nose2 leap.soledad.tests.u1db_tests
diff --git a/run_tests.sh b/run_tests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e36466f84d2c70880a6cd2d8b76cbc57bd9383b3
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+cd common
+python setup.py test
diff --git a/scripts/README.rst b/scripts/README.rst
new file mode 100644
index 0000000000000000000000000000000000000000..fdd1d6427a2fa6820bf1af3d89622a11c1f30fe9
--- /dev/null
+++ b/scripts/README.rst
@@ -0,0 +1,17 @@
+Soledad Scripts
+===============
+
+The scripts in this directory are meant to be used for development purposes.
+
+Currently, the scripts are:
+
+ * server-side-db.py: Gives access to server-side soledad user database,
+ based on the configuration in /etc/leap/soledad-server.conf. One should
+ use it as:
+
+ python -i server-side-db.py <uuid>
+
+ * client-side-db.py: Gives access to client-side soledad user database,
+ based on data stored in ~/.config/leap/soledad. One should use it as:
+
+ python -i client-side-db.py <uuid> <passphrase>
diff --git a/scripts/client-side-db.py b/scripts/client-side-db.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c3df7a4ce010c8681b233798741de68d8003e85
--- /dev/null
+++ b/scripts/client-side-db.py
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+# This script gives client-side access to one Soledad user database by using
+# the data stored in ~/.config/leap/soledad/
+
+import sys
+import os
+
+from leap.common.config import get_path_prefix
+from leap.soledad.client import Soledad
+
+if len(sys.argv) != 3:
+ print 'Usage: %s <uuid> <passphrase>' % sys.argv[0]
+ exit(1)
+
+uuid = sys.argv[1]
+passphrase = unicode(sys.argv[2])
+
+secrets_path = os.path.join(get_path_prefix(), 'leap', 'soledad',
+ '%s.secret' % uuid)
+local_db_path = os.path.join(get_path_prefix(), 'leap', 'soledad',
+ '%s.db' % uuid)
+server_url = 'http://dummy-url'
+cert_file = 'cert'
+
+sol = Soledad(uuid, passphrase, secrets_path, local_db_path, server_url,
+ cert_file)
+db = sol._db
+
+# get replica info
+replica_uid = db._replica_uid
+gen, docs = db.get_all_docs()
+print "replica_uid: %s" % replica_uid
+print "generation: %d" % gen
+gen, trans_id = db._get_generation_info()
+print "transaction_id: %s" % trans_id
diff --git a/scripts/develop_mode.sh b/scripts/develop_mode.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8d2ebaa8e6d52aed20655ef840fcb976656de16e
--- /dev/null
+++ b/scripts/develop_mode.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+cd common
+python setup.py develop
+cd ../client
+python setup.py develop
+cd ../server
+python setup.py develop
diff --git a/scripts/server-side-db.py b/scripts/server-side-db.py
new file mode 100644
index 0000000000000000000000000000000000000000..01a9aaac905ea1bf587f2400b36314d5412983e5
--- /dev/null
+++ b/scripts/server-side-db.py
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+
+# This script gives server-side access to one Soledad user database by using
+# the configuration stored in /etc/leap/soledad-server.conf.
+
+import sys
+from ConfigParser import ConfigParser
+
+from leap.soledad.common.couch import CouchDatabase
+
+if len(sys.argv) != 2:
+ print 'Usage: %s <uuid>' % sys.argv[0]
+ exit(1)
+
+uuid = sys.argv[1]
+
+# get couch url
+cp = ConfigParser()
+cp.read('/etc/leap/soledad-server.conf')
+url = cp.get('soledad-server', 'couch_url')
+
+# access user db
+dbname = 'user-%s' % uuid
+db = CouchDatabase(url, dbname)
+
+# get replica info
+replica_uid = db._replica_uid
+gen, docs = db.get_all_docs()
+print "dbname: %s" % dbname
+print "replica_uid: %s" % replica_uid
+print "generation: %d" % gen
+
+# get relevant docs
+schemes = map(lambda d: d.content['_enc_scheme'], docs)
+pubenc = filter(lambda d: d.content['_enc_scheme'] == 'pubkey', docs)
+
+print "total number of docs: %d" % len(docs)
+print "pubkey encrypted docs: %d" % len(pubenc)