Hello! We are running our annual fundraising. Please consider making a donation if you value this freely available service or want to support people around the world working towards liberatory social change. https://riseup.net/donate.

Commit 5854faf9 authored by Victor's avatar Victor
Browse files

[refactor] merge refactor from drebs

parents 0c22a704 b847d0fa
......@@ -19,7 +19,9 @@ MANIFEST
_trial_temp
.DS_Store
scripts/profiling/sync/profiles
tags
testing/htmlcov
testing/.coverage
testing/test-env
testing/.benchmarks
......@@ -5,3 +5,5 @@ cryptography
pysqlcipher;python_version=="2.7"
pysqlcipher3;python_version=="3.4"
treq
weakref
enum34
......@@ -17,12 +17,14 @@
"""
Soledad - Synchronization Of Locally Encrypted Data Among Devices.
"""
from leap.soledad.client.api import Soledad
from leap.soledad.common import soledad_assert
from .api import Soledad
from ._document import Document, AttachmentStates
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
__all__ = ['soledad_assert', 'Soledad', '__version__']
__all__ = ['soledad_assert', 'Soledad', 'Document', 'AttachmentStates',
'__version__']
......@@ -135,7 +135,7 @@ class SoledadCrypto(object):
and wrapping the result as a simple JSON string with a "raw" key.
:param doc: the document to be encrypted.
:type doc: SoledadDocument
:type doc: Document
:return: A deferred whose callback will be invoked with a JSON string
containing the ciphertext as the value of "raw" key.
:rtype: twisted.internet.defer.Deferred
......@@ -159,7 +159,7 @@ class SoledadCrypto(object):
the decrypted cleartext content from the encrypted document.
:param doc: the document to be decrypted.
:type doc: SoledadDocument
:type doc: Document
:return: The decrypted cleartext content of the document.
:rtype: str
"""
......@@ -225,7 +225,7 @@ def decrypt_sym(data, key, iv, method=ENC_METHOD.aes_256_gcm):
class BlobEncryptor(object):
"""
Produces encrypted data from the cleartext data associated with a given
SoledadDocument using AES-256 cipher in GCM mode.
Document using AES-256 cipher in GCM mode.
The production happens using a Twisted's FileBodyProducer, which uses a
Cooperator to schedule calls and can be paused/resumed. Each call takes at
......
......@@ -30,8 +30,9 @@ from zope.proxy import ProxyBase, setProxiedObject
from leap.soledad.common.log import getLogger
from leap.soledad.common.errors import DatabaseAccessError
from leap.soledad.client import sqlcipher as soledad_sqlcipher
from leap.soledad.client.pragmas import set_init_pragmas
from . import sqlcipher
from . import pragmas
if sys.version_info[0] < 3:
from pysqlcipher import dbapi2
......@@ -73,7 +74,7 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
:rtype: U1DBConnectionPool
"""
if openfun is None and driver == "pysqlcipher":
openfun = partial(set_init_pragmas, opts=opts)
openfun = partial(pragmas.set_init_pragmas, opts=opts)
return U1DBConnectionPool(
opts,
# the following params are relayed "as is" to twisted's
......@@ -87,7 +88,7 @@ class U1DBConnection(adbapi.Connection):
A wrapper for a U1DB connection instance.
"""
u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper
u1db_wrapper = sqlcipher.SoledadSQLCipherWrapper
"""
The U1DB wrapper to use.
"""
......
......@@ -20,8 +20,9 @@ Clientside BlobBackend Storage.
from urlparse import urljoin
import binascii
import errno
import os
import uuid
import base64
from io import BytesIO
......@@ -34,13 +35,18 @@ from twisted.web.client import FileBodyProducer
import treq
from leap.soledad.client.sqlcipher import SQLCipherOptions
from leap.soledad.client import pragmas
from leap.soledad.client._pipes import TruncatedTailPipe, PreamblePipe
from leap.soledad.common.errors import SoledadError
from _crypto import DocInfo, BlobEncryptor, BlobDecryptor
from _http import HTTPClient
from .._document import BlobDoc
from .._crypto import DocInfo
from .._crypto import BlobEncryptor
from .._crypto import BlobDecryptor
from .._http import HTTPClient
from .._pipes import TruncatedTailPipe
from .._pipes import PreamblePipe
from . import pragmas
from . import sqlcipher
logger = Logger()
......@@ -129,6 +135,16 @@ class DecrypterBuffer(object):
return self.decrypter._end_stream(), real_size
def mkdir_p(path):
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
class BlobManager(object):
"""
Ideally, the decrypting flow goes like this:
......@@ -149,6 +165,7 @@ class BlobManager(object):
self, local_path, remote, key, secret, user, token=None,
cert_file=None):
if local_path:
mkdir_p(os.path.dirname(local_path))
self.local = SQLiteBlobBackend(local_path, key)
self.remote = remote
self.secret = secret
......@@ -280,10 +297,13 @@ class SQLiteBlobBackend(object):
def __init__(self, path, key=None):
self.path = os.path.abspath(
os.path.join(path, 'soledad_blob.db'))
mkdir_p(os.path.dirname(self.path))
if not key:
raise ValueError('key cannot be None')
backend = 'pysqlcipher.dbapi2'
opts = SQLCipherOptions('/tmp/ignored', key)
opts = sqlcipher.SQLCipherOptions(
'/tmp/ignored', binascii.b2a_hex(key),
is_raw_key=True, create=True)
pragmafun = partial(pragmas.set_init_pragmas, opts=opts)
openfun = _sqlcipherInitFactory(pragmafun)
......@@ -292,7 +312,11 @@ class SQLiteBlobBackend(object):
cp_openfun=openfun, cp_min=1, cp_max=2, cp_name='blob_pool')
def close(self):
return self.dbpool.close()
from twisted._threads import AlreadyQuit
try:
self.dbpool.close()
except AlreadyQuit:
pass
@defer.inlineCallbacks
def put(self, blob_id, blob_fd, size=None):
......@@ -352,20 +376,6 @@ def _sqlcipherInitFactory(fun):
return _initialize
class BlobDoc(object):
# TODO probably not needed, but convenient for testing for now.
def __init__(self, content, blob_id):
self.blob_id = blob_id
self.is_blob = True
self.blob_fd = content
if blob_id is None:
blob_id = uuid.uuid4().get_hex()
self.blob_id = blob_id
#
# testing facilities
#
......@@ -431,8 +441,7 @@ def testit(reactor):
# TODO convert these into proper unittests
def _manager():
if not os.path.isdir(args.path):
os.makedirs(args.path)
mkdir_p(os.path.dirname(args.path))
manager = BlobManager(
args.path, args.url,
'A' * 32, args.secret,
......
......@@ -50,16 +50,16 @@ from twisted.internet import reactor
from twisted.internet import defer
from twisted.enterprise import adbapi
from leap.soledad.common.document import SoledadDocument
from leap.soledad.common.log import getLogger
from leap.soledad.common.l2db import errors as u1db_errors
from leap.soledad.common.l2db import Document
from leap.soledad.common.l2db.backends import sqlite_backend
from leap.soledad.common.errors import DatabaseAccessError
from leap.soledad.client.http_target import SoledadHTTPSyncTarget
from leap.soledad.client.sync import SoledadSynchronizer
from leap.soledad.client import pragmas
from .._document import Document
from . import sqlite
from . import pragmas
if sys.version_info[0] < 3:
from pysqlcipher import dbapi2 as sqlcipher_dbapi2
......@@ -69,8 +69,8 @@ else:
logger = getLogger(__name__)
# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2
sqlite_backend.dbapi2 = sqlcipher_dbapi2
# Monkey-patch u1db.backends.sqlite with pysqlcipher.dbapi2
sqlite.dbapi2 = sqlcipher_dbapi2
# we may want to collect statistics from the sync process
......@@ -193,7 +193,7 @@ class SQLCipherOptions(object):
# The SQLCipher database
#
class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
class SQLCipherDatabase(sqlite.SQLitePartialExpandDatabase):
"""
A U1DB implementation that uses SQLCipher as its persistence layer.
"""
......@@ -232,7 +232,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
# ---------------------------------------------------------
self._ensure_schema()
self.set_document_factory(soledad_doc_factory)
self.set_document_factory(doc_factory)
self._prime_replica_uid()
def _prime_replica_uid(self):
......@@ -341,7 +341,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
:param doc: The new version of the document.
:type doc: u1db.Document
"""
sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes(
sqlite.SQLitePartialExpandDatabase._put_and_update_indexes(
self, old_doc, doc)
c = self._db_handle.cursor()
c.execute('UPDATE document SET syncable=? WHERE doc_id=?',
......@@ -361,7 +361,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
:return: a Document object.
:type: u1db.Document
"""
doc = sqlite_backend.SQLitePartialExpandDatabase._get_doc(
doc = sqlite.SQLitePartialExpandDatabase._get_doc(
self, doc_id, check_for_conflicts)
if doc:
c = self._db_handle.cursor()
......@@ -437,7 +437,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
self._opts, check_same_thread=False)
self._real_replica_uid = None
self._ensure_schema()
self.set_document_factory(soledad_doc_factory)
self.set_document_factory(doc_factory)
except sqlcipher_dbapi2.DatabaseError as e:
raise DatabaseAccessError(str(e))
......@@ -505,7 +505,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
return self._get_generation()
class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase):
class U1DBSQLiteBackend(sqlite.SQLitePartialExpandDatabase):
"""
A very simple wrapper for u1db around sqlcipher backend.
......@@ -537,7 +537,7 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase):
self._db_handle = conn
self._real_replica_uid = None
self._ensure_schema()
self.set_document_factory(soledad_doc_factory)
self.set_document_factory(doc_factory)
self._prime_replica_uid()
......@@ -562,7 +562,7 @@ def _assert_db_is_encrypted(opts):
# If the regular backend succeeds, then we need to stop because
# the database was not properly initialized.
try:
sqlite_backend.SQLitePartialExpandDatabase(opts.path)
sqlite.SQLitePartialExpandDatabase(opts.path)
except sqlcipher_dbapi2.DatabaseError:
# assert that we can access it using SQLCipher with the given
# key
......@@ -583,17 +583,17 @@ class DatabaseIsNotEncrypted(Exception):
pass
def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
syncable=True):
def doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
syncable=True):
"""
Return a default Soledad Document.
Used in the initialization for SQLCipherDatabase
"""
return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
has_conflicts=has_conflicts, syncable=syncable)
return Document(doc_id=doc_id, rev=rev, json=json,
has_conflicts=has_conflicts, syncable=syncable)
sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase)
sqlite.SQLiteDatabase.register_implementation(SQLCipherDatabase)
#
......
# -*- coding: utf-8 -*-
# _document.py
# Copyright (C) 2017 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Everything related to documents.
"""
import enum
import weakref
import uuid
from twisted.internet import defer
from zope.interface import Interface
from zope.interface import implementer
from leap.soledad.common.document import SoledadDocument
class IDocumentWithAttachment(Interface):
"""
A document that can have an attachment.
"""
def set_store(self, store):
"""
Set the store used by this file to manage attachments.
:param store: The store used to manage attachments.
:type store: Soledad
"""
def put_attachment(self, fd):
"""
Attach data to this document.
Add the attachment to local storage, enqueue for upload.
The document content will be updated with a pointer to the attachment,
but the document has to be manually put in the database to reflect
modifications.
:param fd: A file-like object whose content will be attached to this
document.
:type fd: file-like
:return: A deferred which fires when the attachment has been added to
local storage.
:rtype: Deferred
"""
def get_attachment(self):
"""
Return the data attached to this document.
If document content contains a pointer to the attachment, try to get
the attachment from local storage and, if not found, from remote
storage.
:return: A deferred which fires with a file like-object whose content
is the attachment of this document, or None if nothing is
attached.
:rtype: Deferred
"""
def delete_attachment(self):
"""
Delete the attachment of this document.
The pointer to the attachment will be removed from the document
content, but the document has to be manually put in the database to
reflect modifications.
:return: A deferred which fires when the attachment has been deleted
from local storage.
:rtype: Deferred
"""
def attachment_state(self):
"""
Return the state of the attachment of this document.
The state is a member of AttachmentStates and is of one of NONE,
LOCAL, REMOTE or SYNCED.
:return: A deferred which fires with The state of the attachment of
this document.
:rtype: Deferred
"""
def is_dirty(self):
"""
Return wether this document's content differs from the contents stored
in local database.
:return: Whether this document is dirty or not.
:rtype: bool
"""
def upload_attachment(self):
"""
Upload this document's attachment.
:return: A deferred which fires with the state of the attachment after
it's been uploaded, or NONE if there's no attachment for this
document.
:rtype: Deferred
"""
def download_attachment(self):
"""
Download this document's attachment.
:return: A deferred which fires with the state of the attachment after
it's been downloaded, or NONE if there's no attachment for
this document.
:rtype: Deferred
"""
class BlobDoc(object):
# TODO probably not needed, but convenient for testing for now.
def __init__(self, content, blob_id):
self.blob_id = blob_id
self.is_blob = True
self.blob_fd = content
if blob_id is None:
blob_id = uuid.uuid4().get_hex()
self.blob_id = blob_id
class AttachmentStates(enum.IntEnum):
NONE = 0
LOCAL = 1
REMOTE = 2
SYNCED = 4
@implementer(IDocumentWithAttachment)
class Document(SoledadDocument):
def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False,
syncable=True, store=None):
SoledadDocument.__init__(self, doc_id=doc_id, rev=rev, json=json,
has_conflicts=has_conflicts,
syncable=syncable)
self.set_store(store)
#
# properties
#
@property
def _manager(self):
if not self.store or not hasattr(self.store, 'blobmanager'):
raise Exception('No blob manager found to manage attachments.')
return self.store.blobmanager
@property
def _blob_id(self):
if self.content and 'blob_id' in self.content:
return self.content['blob_id']
return None
def get_store(self):
return self._store() if self._store else None
def set_store(self, store):
self._store = weakref.ref(store) if store else None
store = property(get_store, set_store)
#
# attachment api
#
def put_attachment(self, fd):
# add pointer to content
blob_id = self._blob_id or str(uuid.uuid4())
if not self.content:
self.content = {}
self.content['blob_id'] = blob_id
# put using manager
blob = BlobDoc(fd, blob_id)
fd.seek(0, 2)
size = fd.tell()
fd.seek(0)
return self._manager.put(blob, size)
def get_attachment(self):
if not self._blob_id:
return defer.succeed(None)
return self._manager.get(self._blob_id)
def delete_attachment(self):
raise NotImplementedError
@defer.inlineCallbacks
def attachment_state(self):
state = AttachmentStates.NONE
if not self._blob_id:
defer.returnValue(state)
local_list = yield self._manager.local_list()
if self._blob_id in local_list:
state |= AttachmentStates.LOCAL
remote_list = yield self._manager.remote_list()
if self._blob_id in remote_list:
state |= AttachmentStates.REMOTE
defer.returnValue(state)
@defer.inlineCallbacks
def is_dirty(self):
stored = yield self.store.get_doc(self.doc_id)
if stored.content != self.content:
defer.returnValue(True)
defer.returnValue(False)
@defer.inlineCallbacks
def upload_attachment(self):
if not self._blob_id:
defer.returnValue(AttachmentStates.NONE)
fd = yield self._manager.get_blob(self._blob_id)
# TODO: turn following method into a public one
yield self._manager._encrypt_and_upload(self._blob_id, fd)
defer.returnValue(self.attachment_state())
@defer.inlineCallbacks
def download_attachment(self):
if not self._blob_id:
defer.returnValue(None)
yield self.get_attachment()
defer.returnValue(self.attachment_state())
......@@ -23,8 +23,8 @@ from hashlib import sha256
from leap.soledad.common import SHARED_DB_NAME
from leap.soledad.common.log import getLogger
from leap.soledad.common.document import SoledadDocument
from leap.soledad.client.shared_db import SoledadSharedDatabase
from leap.soledad.client._document import Document
from leap.soledad.client._secrets.util import emit, UserDataMixin
......@@ -111,7 +111,7 @@ class SecretsStorage(UserDataMixin):
def save_remote(self, encrypted):
doc = self._remote_doc
if not doc:
doc = SoledadDocument(doc_id=self._remote_doc_id())
doc = Document(doc_id=self._remote_doc_id())
doc.content = encrypted
db = self._shared_db
if not db:
......
......@@ -51,13 +51,14 @@ from leap.soledad.common.l2db.remote import http_client